transformer/[LangCon2020]_Transformer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "[LangCon2020] Transformer.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "dg6iIMEJRVF5",
        "Aw_RiCrZd1Jp",
        "pKEZAHeCzxRm",
        "6zIuHDPU0qif",
        "oNFjgMAqf5F9",
        "8AaJfELdJAws",
        "v4JrzZa-JFqz",
        "5F2o9a06JKXK",
        "4Igt7GwvJWgV",
        "SGHIBE4SUYtC",
        "aw2CjyEzJdCu",
        "KlbxVAZapnGy",
        "rRIB0gjRJjAU",
        "69ZQF9_QJtgD",
        "kRJOG3puSMbl",
        "sC6NSo3cJ2HE",
        "TnFBTppYJ4QE",
        "253aegyMJ5s9",
        "1FbtXvu_S6Hm",
        "H1bAp9rqKAMW",
        "oZcLQR0NKINv",
        "PTpTinUoKLc1",
        "wWAtDOgtLHZr",
        "owr3DtdjLHZs",
        "UP9HqezULHaC",
        "QJ4wDtrPLHaE",
        "ZZYToSV7yRVF",
        "VlhvQ5TcKlp_",
        "O33u8_SWKrH4",
        "oPBEA8FAl39-",
        "64BhMxZ4KUI1",
        "SbQ-4OZ2KdbH",
        "sc0xdcL-6ZZl",
        "W_ks6mcOKzow",
        "hXWYTm6JK9OR",
        "f9HRMhWKK-6I",
        "Z3Jc-KlSQDMG",
        "OnCeiVe2NrNz",
        "xDuEmCeoN2lq",
        "5rTITKdIN5nU",
        "oCkXJQIUlETb",
        "eZXtO5pKONH1",
        "F8_N3eEDOA2z",
        "I9zNYIgzPwW3",
        "teJEuUHAlI5f",
        "L9be5hBhR0k9",
        "XpS8LxinR4HZ",
        "hsohSU2sR7QF",
        "fC79J4CYuWF7",
        "MRwW6_IRTtWv",
        "fQDSL7Q1TrvJ",
        "_N5l-XL5WAuP",
        "tOZoGy998QRV",
        "34abQnr98r7B",
        "S3266YeXD2Yw",
        "1-xIfzOMF8nO",
        "xclGDmnmDCUY",
        "hjqOTrjvxji7",
        "6x6W5AYLaqMw",
        "TOWlO39Vaoit",
        "ZV5zGuQQL_mK",
        "fAR-ddNrMGby",
        "jWIUoq0cbYT8",
        "qQqF2KJVcLBF",
        "DYYcdK-_MRni",
        "Ey3sghogI4rz",
        "VKkOrvxoJM1h",
        "A7k-JyedJK6J",
        "HNHVGEgyacKg",
        "KMtlbz8YMd_j",
        "dtqNHlkRdjz5"
      ],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "b7e44e68f53c4c869dcfbcdd1e745143": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d004b3226eaa4eed8cf18901fde5fb8d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c374c11d8f4b4ac6bbaa50abc233ceb8",
              "IPY_MODEL_d25a3e30f3584961afdd0ebfd7009d99"
            ]
          }
        },
        "d004b3226eaa4eed8cf18901fde5fb8d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c374c11d8f4b4ac6bbaa50abc233ceb8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8d0bf43a347e4bada53a1d448fe9c4d0",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 11823,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 11823,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2f9d1ec2fa6947ceb8ba53a10fdb314c"
          }
        },
        "d25a3e30f3584961afdd0ebfd7009d99": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ba2f186f0252446cbcdeb0253939b249",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 11823/11823 [00:03&lt;00:00, 3203.39it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_029631a37ac343ab946110e0a0939fa4"
          }
        },
        "8d0bf43a347e4bada53a1d448fe9c4d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2f9d1ec2fa6947ceb8ba53a10fdb314c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ba2f186f0252446cbcdeb0253939b249": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "029631a37ac343ab946110e0a0939fa4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dg6iIMEJRVF5",
        "colab_type": "text"
      },
      "source": [
        "# 환경"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LdVt9pWh8eYV",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "outputId": "85def9a3-4748-465b-e473-9c370e80b776"
      },
      "source": [
        "# vocab 사용을 위한 sentencepiece 설치\n",
        "!pip install sentencepiece"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
            "\r\u001b[K     |▎                               | 10kB 20.8MB/s eta 0:00:01\r\u001b[K     |▋                               | 20kB 3.5MB/s eta 0:00:01\r\u001b[K     |█                               | 30kB 4.7MB/s eta 0:00:01\r\u001b[K     |█▏                              | 40kB 5.0MB/s eta 0:00:01\r\u001b[K     |█▌                              | 51kB 4.1MB/s eta 0:00:01\r\u001b[K     |█▉                              | 61kB 4.6MB/s eta 0:00:01\r\u001b[K     |██▏                             | 71kB 5.0MB/s eta 0:00:01\r\u001b[K     |██▍                             | 81kB 5.4MB/s eta 0:00:01\r\u001b[K     |██▊                             | 92kB 5.7MB/s eta 0:00:01\r\u001b[K     |███                             | 102kB 5.6MB/s eta 0:00:01\r\u001b[K     |███▍                            | 112kB 5.6MB/s eta 0:00:01\r\u001b[K     |███▋                            | 122kB 5.6MB/s eta 0:00:01\r\u001b[K     |████                            | 133kB 5.6MB/s eta 0:00:01\r\u001b[K     |████▎                           | 143kB 5.6MB/s eta 0:00:01\r\u001b[K     |████▋                           | 153kB 5.6MB/s eta 0:00:01\r\u001b[K     |████▉                           | 163kB 5.6MB/s eta 0:00:01\r\u001b[K     |█████▏                          | 174kB 5.6MB/s eta 0:00:01\r\u001b[K     |█████▌                          | 184kB 5.6MB/s eta 0:00:01\r\u001b[K     |█████▉                          | 194kB 5.6MB/s eta 0:00:01\r\u001b[K     |██████                          | 204kB 5.6MB/s eta 0:00:01\r\u001b[K     |██████▍                         | 215kB 5.6MB/s eta 0:00:01\r\u001b[K     |██████▊                         | 225kB 5.6MB/s eta 0:00:01\r\u001b[K     |███████                         | 235kB 5.6MB/s eta 0:00:01\r\u001b[K     |███████▎                        | 245kB 5.6MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 256kB 5.6MB/s eta 0:00:01\r\u001b[K     |████████                        | 266kB 5.6MB/s eta 0:00:01\r\u001b[K     |████████▎                       | 276kB 5.6MB/s eta 0:00:01\r\u001b[K     |████████▌                       | 286kB 5.6MB/s eta 0:00:01\r\u001b[K     |████████▉                       | 296kB 5.6MB/s eta 0:00:01\r\u001b[K     |█████████▏                      | 307kB 5.6MB/s eta 0:00:01\r\u001b[K     |█████████▌                      | 317kB 5.6MB/s eta 0:00:01\r\u001b[K     |█████████▊                      | 327kB 5.6MB/s eta 0:00:01\r\u001b[K     |██████████                      | 337kB 5.6MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 348kB 5.6MB/s eta 0:00:01\r\u001b[K     |██████████▊                     | 358kB 5.6MB/s eta 0:00:01\r\u001b[K     |███████████                     | 368kB 5.6MB/s eta 0:00:01\r\u001b[K     |███████████▎                    | 378kB 5.6MB/s eta 0:00:01\r\u001b[K     |███████████▋                    | 389kB 5.6MB/s eta 0:00:01\r\u001b[K     |████████████                    | 399kB 5.6MB/s eta 0:00:01\r\u001b[K     |████████████▏                   | 409kB 5.6MB/s eta 0:00:01\r\u001b[K     |████████████▌                   | 419kB 5.6MB/s eta 0:00:01\r\u001b[K     |████████████▉                   | 430kB 5.6MB/s eta 0:00:01\r\u001b[K     |█████████████▏                  | 440kB 5.6MB/s eta 0:00:01\r\u001b[K     |█████████████▍                  | 450kB 5.6MB/s eta 0:00:01\r\u001b[K     |█████████████▊                  | 460kB 5.6MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 471kB 5.6MB/s eta 0:00:01\r\u001b[K     |██████████████▍                 | 481kB 5.6MB/s eta 0:00:01\r\u001b[K     |██████████████▋                 | 491kB 5.6MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 501kB 5.6MB/s eta 0:00:01\r\u001b[K     |███████████████▎                | 512kB 5.6MB/s eta 0:00:01\r\u001b[K     |███████████████▋                | 522kB 5.6MB/s eta 0:00:01\r\u001b[K     |███████████████▉                | 532kB 5.6MB/s eta 0:00:01\r\u001b[K     |████████████████▏               | 542kB 5.6MB/s eta 0:00:01\r\u001b[K     |████████████████▌               | 552kB 5.6MB/s eta 0:00:01\r\u001b[K     |████████████████▉               | 563kB 5.6MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 573kB 5.6MB/s eta 0:00:01\r\u001b[K     |█████████████████▍              | 583kB 5.6MB/s eta 0:00:01\r\u001b[K     |█████████████████▊              | 593kB 5.6MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 604kB 5.6MB/s eta 0:00:01\r\u001b[K     |██████████████████▎             | 614kB 5.6MB/s eta 0:00:01\r\u001b[K     |██████████████████▋             | 624kB 5.6MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 634kB 5.6MB/s eta 0:00:01\r\u001b[K     |███████████████████▎            | 645kB 5.6MB/s eta 0:00:01\r\u001b[K     |███████████████████▌            | 655kB 5.6MB/s eta 0:00:01\r\u001b[K     |███████████████████▉            | 665kB 5.6MB/s eta 0:00:01\r\u001b[K     |████████████████████▏           | 675kB 5.6MB/s eta 0:00:01\r\u001b[K     |████████████████████▌           | 686kB 5.6MB/s eta 0:00:01\r\u001b[K     |████████████████████▊           | 696kB 5.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 706kB 5.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████▍          | 716kB 5.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████▋          | 727kB 5.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 737kB 5.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████▎         | 747kB 5.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████▋         | 757kB 5.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████▉         | 768kB 5.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████▏        | 778kB 5.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████▌        | 788kB 5.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 798kB 5.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 808kB 5.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████▍       | 819kB 5.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████▊       | 829kB 5.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 839kB 5.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▎      | 849kB 5.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▋      | 860kB 5.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 870kB 5.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▎     | 880kB 5.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▌     | 890kB 5.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▉     | 901kB 5.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▏    | 911kB 5.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▌    | 921kB 5.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▊    | 931kB 5.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 942kB 5.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▍   | 952kB 5.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▊   | 962kB 5.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 972kB 5.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▎  | 983kB 5.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▋  | 993kB 5.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 1.0MB 5.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▏ | 1.0MB 5.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▌ | 1.0MB 5.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▉ | 1.0MB 5.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▏| 1.0MB 5.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▍| 1.1MB 5.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▊| 1.1MB 5.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 1.1MB 5.6MB/s \n",
            "\u001b[?25hInstalling collected packages: sentencepiece\n",
            "Successfully installed sentencepiece-0.1.91\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BQCFd-7JRcca",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# imports\n",
        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
        "\n",
        "import tensorflow as tf\n",
        "import tensorflow.keras.backend as K\n",
        "\n",
        "import os\n",
        "import re\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import random\n",
        "import collections\n",
        "import json\n",
        "from datetime import datetime\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import sentencepiece as spm\n",
        "from tqdm.notebook import tqdm\n",
        "\n",
        "random_seed = 1234\n",
        "random.seed(random_seed)\n",
        "np.random.seed(random_seed)\n",
        "tf.random.set_seed(random_seed)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-vVvopnXRgka",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        },
        "outputId": "2520ecb2-7a1e-465d-ab6a-2906a46dffe7"
      },
      "source": [
        "# gpu 확인\n",
        "!nvidia-smi"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sun Aug 23 13:12:24 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 450.57       Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   53C    P8    10W /  70W |      0MiB / 15079MiB |      0%      Default |\n",
            "|                               |                      |                 ERR! |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FlYm3JM7fcdd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "e4dcfbd4-a902-4efd-fe74-506b61e822cb"
      },
      "source": [
        "# tf version 및 gpu 확인\n",
        "print(tf.__version__)\n",
        "print(tf.config.list_physical_devices('GPU'))\n",
        "print(tf.test.gpu_device_name())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.3.0\n",
            "[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]\n",
            "/device:GPU:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Aw_RiCrZd1Jp",
        "colab_type": "text"
      },
      "source": [
        "# Transformer Tutorial"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pKEZAHeCzxRm",
        "colab_type": "text"
      },
      "source": [
        "## Config"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u9VJvPSx12Q8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Config(dict):\n",
        "    \"\"\"\n",
        "    json을 config 형태로 사용하기 위한 Class\n",
        "    :param dict: config dictionary\n",
        "    \"\"\"\n",
        "    __getattr__ = dict.__getitem__\n",
        "    __setattr__ = dict.__setitem__\n",
        "\n",
        "    @classmethod\n",
        "    def load(cls, file):\n",
        "        \"\"\"\n",
        "        file에서 Config를 생성 함\n",
        "        :param file: filename\n",
        "        \"\"\"\n",
        "        with open(file, 'r') as f:\n",
        "            config = json.loads(f.read())\n",
        "            return Config(config)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Bb_Q7DhzlGp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0c78160e-1e75-49d5-9ea6-4eafed54c617"
      },
      "source": [
        "# config 생성\n",
        "# d_model: model hidden dim\n",
        "# n_head: multi head attention head number\n",
        "# d_head: multi head attention head dim\n",
        "# dropout: dropout rate\n",
        "# d_ff: feed forward dim\n",
        "# layernorm_epsilon: layernormal epsilon\n",
        "# n_layer: layer number\n",
        "# n_seq: sequence max number\n",
        "# n_vocab: vocab count\n",
        "# i_pad: vocab pad id\n",
        "config = Config({\"d_model\": 8, \"n_head\": 2, \"d_head\": 4, \"dropout\": 0.1, \"d_ff\": 32, \"layernorm_epsilon\": 0.001, \"n_layer\": 3, \"n_seq\": 10, \"n_vocab\": 0, \"i_pad\": 0})\n",
        "config.n_vocab = 10\n",
        "config.i_pad = 0\n",
        "config"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'d_ff': 32,\n",
              " 'd_head': 4,\n",
              " 'd_model': 8,\n",
              " 'dropout': 0.1,\n",
              " 'i_pad': 0,\n",
              " 'layernorm_epsilon': 0.001,\n",
              " 'n_head': 2,\n",
              " 'n_layer': 3,\n",
              " 'n_seq': 10,\n",
              " 'n_vocab': 10}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6zIuHDPU0qif",
        "colab_type": "text"
      },
      "source": [
        "## Inputs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yP8BiMdU0pjZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "88194a2b-ff32-4f0d-86a2-7968331bcd34"
      },
      "source": [
        "# encoder 입력 값 정의\n",
        "enc_tokens = np.random.randint(1, config.n_vocab, (1, 6))\n",
        "# 마지막은 padding\n",
        "enc_tokens[:,5:] = config.i_pad\n",
        "enc_tokens"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[4, 7, 6, 5, 9, 0]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GkZb4b9-00Ng",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "115cf449-3d35-48a9-af4a-391a0253ac14"
      },
      "source": [
        "# 입력 값 정의\n",
        "dec_tokens = np.random.randint(1, config.n_vocab, (1, 5))\n",
        "# 마지막은 padding\n",
        "dec_tokens[:,4:] = config.i_pad\n",
        "dec_tokens"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[8, 7, 9, 1, 0]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oNFjgMAqf5F9",
        "colab_type": "text"
      },
      "source": [
        "## Weight Shared Embedding\n",
        "\n",
        "![](https://paul-hyun.github.io/assets/transformer/weight_shared_embedding.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8AaJfELdJAws",
        "colab_type": "text"
      },
      "source": [
        "### Tutorial Embedding"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rCGzHgobgGju",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0b596f38-d6dc-4a00-ac7c-dfcf23cccb12"
      },
      "source": [
        "# embedding weight 초기화\n",
        "embed_weight = tf.random.normal([config.n_vocab, config.d_model])\n",
        "embed_weight"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(10, 8), dtype=float32, numpy=\n",
              "array([[ 0.8369314 , -0.73429775,  1.0402943 ,  0.04035992, -0.72186583,\n",
              "         1.0794858 ,  0.9032698 , -0.73601735],\n",
              "       [-0.36105633, -0.6078763 ,  0.07614239, -0.7211218 ,  0.14663379,\n",
              "         1.0210938 , -0.17570543,  1.2770809 ],\n",
              "       [ 0.15558743,  0.11838357, -0.10230648,  0.02042525,  0.12023098,\n",
              "        -1.7356888 , -0.37274906, -0.8804749 ],\n",
              "       [ 0.8006071 , -1.0832181 , -0.8637153 ,  0.13195874,  0.44462258,\n",
              "         1.1913583 ,  1.5747397 , -0.57046455],\n",
              "       [ 1.4189912 ,  0.5309418 , -0.3397897 , -0.5185629 , -1.1675293 ,\n",
              "         0.77573264, -1.5179933 ,  0.73918307],\n",
              "       [ 1.8952272 , -0.20591696, -0.72119623, -1.1176348 ,  0.85716414,\n",
              "        -0.37806422, -0.121459  ,  0.00689717],\n",
              "       [ 1.2772721 ,  0.3844953 ,  0.18691574,  1.2595506 , -0.15900195,\n",
              "         0.1210066 , -0.42040455,  2.4253113 ],\n",
              "       [-0.3567563 , -0.61342746,  1.3244288 , -0.5528212 ,  1.2431467 ,\n",
              "        -0.24536318,  0.28992933, -0.25124574],\n",
              "       [-1.3603296 , -0.09727097, -0.8926318 ,  0.3699977 , -0.43288955,\n",
              "         0.44764125,  0.54849035, -0.74648273],\n",
              "       [ 0.18089715, -0.19718426, -0.24024896,  0.8510773 ,  1.5659981 ,\n",
              "        -1.1252567 , -1.9409475 ,  1.5495228 ]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9o24Kl2ZgLnR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f0305218-407a-4589-d7fd-f8bdb17b2f40"
      },
      "source": [
        "# embedding lookup 대신에 tf.gather 함수 사용\n",
        "token_embed1 = tf.gather(embed_weight, enc_tokens)\n",
        "token_embed1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 8), dtype=float32, numpy=\n",
              "array([[[ 1.4189912 ,  0.5309418 , -0.3397897 , -0.5185629 ,\n",
              "         -1.1675293 ,  0.77573264, -1.5179933 ,  0.73918307],\n",
              "        [-0.3567563 , -0.61342746,  1.3244288 , -0.5528212 ,\n",
              "          1.2431467 , -0.24536318,  0.28992933, -0.25124574],\n",
              "        [ 1.2772721 ,  0.3844953 ,  0.18691574,  1.2595506 ,\n",
              "         -0.15900195,  0.1210066 , -0.42040455,  2.4253113 ],\n",
              "        [ 1.8952272 , -0.20591696, -0.72119623, -1.1176348 ,\n",
              "          0.85716414, -0.37806422, -0.121459  ,  0.00689717],\n",
              "        [ 0.18089715, -0.19718426, -0.24024896,  0.8510773 ,\n",
              "          1.5659981 , -1.1252567 , -1.9409475 ,  1.5495228 ],\n",
              "        [ 0.8369314 , -0.73429775,  1.0402943 ,  0.04035992,\n",
              "         -0.72186583,  1.0794858 ,  0.9032698 , -0.73601735]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OwUzEGoVgU37",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "062d3253-9693-4624-d97d-d4b0da2c534e"
      },
      "source": [
        "# embedding 값에 d_model ** 0.5 곱\n",
        "token_embed = token_embed1 * config.d_model ** 0.5\n",
        "token_embed"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 8), dtype=float32, numpy=\n",
              "array([[[ 4.013513  ,  1.5017301 , -0.96107036, -1.4667174 ,\n",
              "         -3.3022716 ,  2.1941032 , -4.2935333 ,  2.0907254 ],\n",
              "        [-1.0090592 , -1.7350348 ,  3.7460504 , -1.5636145 ,\n",
              "          3.5161498 , -0.69399184,  0.820044  , -0.71063024],\n",
              "        [ 3.612671  ,  1.0875169 ,  0.5286775 ,  3.562547  ,\n",
              "         -0.44972542,  0.34225833, -1.1890836 ,  6.859816  ],\n",
              "        [ 5.360512  , -0.58242106, -2.039851  , -3.1611485 ,\n",
              "          2.4244263 , -1.0693271 , -0.34353793,  0.01950813],\n",
              "        [ 0.5116544 , -0.5577213 , -0.6795267 ,  2.40721   ,\n",
              "          4.4293113 , -3.1827064 , -5.4898286 ,  4.382712  ],\n",
              "        [ 2.3671994 , -2.0769076 ,  2.9423966 ,  0.11415508,\n",
              "         -2.041745  ,  3.0532467 ,  2.554833  , -2.0817714 ]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v4JrzZa-JFqz",
        "colab_type": "text"
      },
      "source": [
        "### Tutorial Linear"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HDIsy0HogjvO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c60ec82d-6f11-4ee0-fa26-f9bfece8a637"
      },
      "source": [
        "# linear layer\n",
        "# (n_batch, n_seq, d_model) -> (n_batch * n_seq, d_model)\n",
        "n_batch, n_seq, d_model = tf.shape(token_embed)\n",
        "token_reshape = tf.reshape(token_embed, [-1, d_model])\n",
        "token_reshape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(6, 8), dtype=float32, numpy=\n",
              "array([[ 4.013513  ,  1.5017301 , -0.96107036, -1.4667174 , -3.3022716 ,\n",
              "         2.1941032 , -4.2935333 ,  2.0907254 ],\n",
              "       [-1.0090592 , -1.7350348 ,  3.7460504 , -1.5636145 ,  3.5161498 ,\n",
              "        -0.69399184,  0.820044  , -0.71063024],\n",
              "       [ 3.612671  ,  1.0875169 ,  0.5286775 ,  3.562547  , -0.44972542,\n",
              "         0.34225833, -1.1890836 ,  6.859816  ],\n",
              "       [ 5.360512  , -0.58242106, -2.039851  , -3.1611485 ,  2.4244263 ,\n",
              "        -1.0693271 , -0.34353793,  0.01950813],\n",
              "       [ 0.5116544 , -0.5577213 , -0.6795267 ,  2.40721   ,  4.4293113 ,\n",
              "        -3.1827064 , -5.4898286 ,  4.382712  ],\n",
              "       [ 2.3671994 , -2.0769076 ,  2.9423966 ,  0.11415508, -2.041745  ,\n",
              "         3.0532467 ,  2.554833  , -2.0817714 ]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rsrpvA1qgz8e",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c0a1567e-4b9c-47ff-9ca5-e14c78e306e8"
      },
      "source": [
        "# matrix mul with embd_weight\n",
        "token_linear = tf.matmul(token_reshape, embed_weight, transpose_b=True)\n",
        "token_reshape.shape, embed_weight.shape, token_linear"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(TensorShape([6, 8]),\n",
              " TensorShape([10, 8]),\n",
              " <tf.Tensor: shape=(6, 10), dtype=float32, numpy=\n",
              " array([[  0.5325967 ,   3.8031166 ,  -3.5751386 ,  -4.585093  ,\n",
              "          21.20014   ,   6.50547   ,  11.342962  ,  -9.228752  ,\n",
              "          -6.794538  ,   3.345391  ],\n",
              "        [  2.2398207 ,   1.5871406 ,   1.1697465 ,   0.06301922,\n",
              "          -9.228752  ,   0.6625772 ,  -5.936514  ,  12.207751  ,\n",
              "          -3.23346   ,   1.5232466 ],\n",
              "        [ -2.5101416 ,   4.7587724 ,  -5.535282  ,  -3.8501906 ,\n",
              "          11.342962  ,   1.9368417 ,  26.868544  ,  -5.936514  ,\n",
              "          -9.599031  ,  15.192061  ],\n",
              "        [ -0.5646777 ,  -0.10825825,   3.1675892 ,   5.5191536 ,\n",
              "           6.50547   ,  17.807726  ,   1.9368415 ,   0.6625773 ,\n",
              "          -8.315366  ,   4.581167  ],\n",
              "        [-14.589603  ,   2.3279722 ,   4.376459  , -11.049267  ,\n",
              "           3.3453908 ,   4.5811677 ,  15.19206   ,   1.5232463 ,\n",
              "          -8.769389  ,  30.378742  ],\n",
              "        [ 15.181544  ,   0.26030588,  -4.8405914 ,   9.55909   ,\n",
              "           0.53259647,  -0.5646776 ,  -2.5101418 ,   2.2398207 ,\n",
              "          -0.3964715 , -14.589603  ]], dtype=float32)>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2J-GDv21jO-3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9f75efaa-9891-4987-f41f-8c7fcbd2db8d"
      },
      "source": [
        "# (bs * n_seq, n_vocab) -> (bs, n_seq, n_vocab)\n",
        "output = tf.reshape(token_linear, [n_batch, n_seq, config.n_vocab])\n",
        "output.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TensorShape([1, 6, 10])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5F2o9a06JKXK",
        "colab_type": "text"
      },
      "source": [
        "### Class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HadAObQ2i6FZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def kernel_initializer(stddev=0.02):\n",
        "    \"\"\"\n",
        "    parameter initializer 생성\n",
        "    :param stddev: 생성할 랜덤 변수의 표준편차\n",
        "    \"\"\"\n",
        "    return tf.keras.initializers.TruncatedNormal(stddev=stddev)\n",
        "\n",
        "\n",
        "class SharedEmbedding(tf.keras.layers.Layer):\n",
        "    \"\"\"\n",
        "    Weighed Shaed Embedding Class\n",
        "    \"\"\"\n",
        "    def __init__(self, config, name=\"weight_shared_embedding\"):\n",
        "        \"\"\"\n",
        "        생성자\n",
        "        :param config: Config 객체\n",
        "        :param name: layer name\n",
        "        \"\"\"\n",
        "        super().__init__(name=name)\n",
        "\n",
        "        self.n_vocab = config.n_vocab\n",
        "        self.d_model = config.d_model\n",
        "    \n",
        "    def build(self, input_shape):\n",
        "        \"\"\"\n",
        "        shared weight 생성\n",
        "        :param input_shape: Tensor Shape (not used)\n",
        "        \"\"\"\n",
        "        with tf.name_scope(\"shared_embedding_weight\"):\n",
        "            self.shared_weights = self.add_weight(\n",
        "                \"weights\",\n",
        "                shape=[self.n_vocab, self.d_model],\n",
        "                initializer=kernel_initializer(stddev=self.d_model ** -0.5)\n",
        "            )\n",
        "\n",
        "    def call(self, inputs, mode=\"embedding\"):\n",
        "        \"\"\"\n",
        "        layer 실행\n",
        "        :param inputs: 입력\n",
        "        :param mode: 실행 모드\n",
        "        :return: embedding or linear 실행 결과\n",
        "        \"\"\"\n",
        "        # mode가 embedding일 경우 embedding lookup 실행\n",
        "        if mode == \"embedding\":\n",
        "            return self._embedding(inputs)\n",
        "        # mode가 linear일 경우 linear 실행\n",
        "        elif mode == \"linear\":\n",
        "            return self._linear(inputs)\n",
        "        # mode가 기타일 경우 오류 발생\n",
        "        else:\n",
        "            raise ValueError(f\"mode {mode} is not valid.\")\n",
        "    \n",
        "    def _embedding(self, inputs):\n",
        "        \"\"\"\n",
        "        embedding lookup\n",
        "        :param inputs: 입력\n",
        "        \"\"\"\n",
        "        embed = tf.gather(self.shared_weights, tf.cast(inputs, tf.int32))\n",
        "        embed *= self.d_model ** 0.5\n",
        "        return embed\n",
        "\n",
        "    def _linear(self, inputs):  # (bs, n_seq, d_model)\n",
        "        \"\"\"\n",
        "        linear 실행\n",
        "        :param inputs: 입력\n",
        "        \"\"\"\n",
        "        n_batch = tf.shape(inputs)[0]\n",
        "        n_seq = tf.shape(inputs)[1]\n",
        "        inputs = tf.reshape(inputs, [-1, self.d_model])  # (bs * n_seq, d_model)\n",
        "        outputs = tf.matmul(inputs, self.shared_weights, transpose_b=True)\n",
        "        outputs = tf.reshape(outputs, [n_batch, n_seq, self.n_vocab])  # (bs, n_seq, n_vocab)\n",
        "        return outputs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4Igt7GwvJWgV",
        "colab_type": "text"
      },
      "source": [
        "### Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mkDCszl4kT_8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a15eec7f-3a95-41a7-84a5-5189ad7eb0b5"
      },
      "source": [
        "# SharedEmbedding embedding test: shape이 (bs, n_seq, d_model)이면 성공\n",
        "embedding = SharedEmbedding(config)\n",
        "\n",
        "enc_embed = embedding(enc_tokens)\n",
        "assert enc_embed.shape == (enc_tokens.shape[0], enc_tokens.shape[1], config.d_model)\n",
        "enc_embed"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 8), dtype=float32, numpy=\n",
              "array([[[ 1.0438616 ,  0.77873224,  0.3661703 ,  0.04562705,\n",
              "         -1.1118658 ,  0.83142024, -0.50751346, -1.6061306 ],\n",
              "        [-1.488764  ,  0.66749793,  0.45289573, -1.3284955 ,\n",
              "         -0.8350257 , -0.14791092,  0.46348926,  0.68240684],\n",
              "        [ 0.53093785,  1.8387237 , -1.4564208 , -0.22360843,\n",
              "          0.8672876 , -0.75745213,  1.4629304 , -1.8092779 ],\n",
              "        [-0.13200705,  0.9998998 , -0.9900697 , -0.2264822 ,\n",
              "          0.5969744 , -0.13854134,  1.1348592 , -0.04831744],\n",
              "        [-1.3377118 ,  1.3356653 , -0.26913092, -1.4376115 ,\n",
              "         -0.3685191 ,  1.0541942 ,  0.37812105, -0.3294939 ],\n",
              "        [ 1.1468066 ,  0.96459925,  0.87830144, -0.31010216,\n",
              "         -0.4058371 , -0.04709157, -0.32690668,  0.5287341 ]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tg0vVqkx2jFb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "83240f4e-768b-4bfb-ae85-a6aed49186d0"
      },
      "source": [
        "# SharedEmbedding embedding test: shape이 (bs, n_seq, d_model)이면 성공\n",
        "dec_embed = embedding(dec_tokens)\n",
        "assert dec_embed.shape == (dec_tokens.shape[0], dec_tokens.shape[1], config.d_model)\n",
        "dec_embed"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 8), dtype=float32, numpy=\n",
              "array([[[ 1.7347238 , -0.40107146,  1.5544791 ,  0.83802575,\n",
              "         -0.6956781 , -0.06145862,  1.496493  , -1.8350446 ],\n",
              "        [-1.488764  ,  0.66749793,  0.45289573, -1.3284955 ,\n",
              "         -0.8350257 , -0.14791092,  0.46348926,  0.68240684],\n",
              "        [-1.3377118 ,  1.3356653 , -0.26913092, -1.4376115 ,\n",
              "         -0.3685191 ,  1.0541942 ,  0.37812105, -0.3294939 ],\n",
              "        [ 0.47501525, -0.7355799 ,  0.66509944, -0.3943125 ,\n",
              "          0.71738523, -0.24470258, -0.27618337,  0.6821183 ],\n",
              "        [ 1.1468066 ,  0.96459925,  0.87830144, -0.31010216,\n",
              "         -0.4058371 , -0.04709157, -0.32690668,  0.5287341 ]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ncgRJ-AJklJK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0543d308-5724-43eb-d00b-f39529f208c1"
      },
      "source": [
        "# SharedEmbedding linear test: shape에 (bs, n_seq, n_vocab)이면 성공\n",
        "linear_outputs = embedding(dec_embed, mode=\"linear\")\n",
        "assert linear_outputs.shape == (dec_tokens.shape[0], dec_tokens.shape[1], config.n_vocab)\n",
        "linear_outputs"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 10), dtype=float32, numpy=\n",
              "array([[[ 0.5422481 , -0.1154615 ,  0.07719773, -1.3376666 ,\n",
              "          1.7734799 , -0.34601617,  0.9492115 , -1.1413567 ,\n",
              "          4.378208  , -1.1021316 ],\n",
              "        [ 0.10657363, -0.21159723,  0.99548954,  0.49363035,\n",
              "         -0.514364  ,  0.25861242, -0.38693726,  2.1325045 ,\n",
              "         -1.1413567 ,  1.6876042 ],\n",
              "        [-0.08280095, -0.73594666,  2.20541   , -0.30941206,\n",
              "          0.39001492,  0.77186346,  0.88045573,  1.6876042 ,\n",
              "         -1.1021316 ,  2.549582  ],\n",
              "        [ 0.252074  ,  0.87703884, -0.9234558 ,  0.5945406 ,\n",
              "         -0.63919353, -0.44251698, -0.99400353, -0.21159722,\n",
              "         -0.11546156, -0.73594666],\n",
              "        [ 1.2963187 ,  0.25207406, -0.58869195, -0.17177188,\n",
              "          0.70162946, -0.21868029, -0.20452958,  0.10657358,\n",
              "          0.5422482 , -0.08280102]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SGHIBE4SUYtC",
        "colab_type": "text"
      },
      "source": [
        "## Postiona Encoding\n",
        "\n",
        "![](https://paul-hyun.github.io/assets/transformer/position_encoding.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aw2CjyEzJdCu",
        "colab_type": "text"
      },
      "source": [
        "### Tutorial Encoding"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HBBT_PR99-yA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ae3dcafb-3aa2-4f59-b20c-712015a0f27e"
      },
      "source": [
        "# 지수 계산\n",
        "exs = [2 * (i_ang // 2) / config.d_model for i_ang in range(config.d_model)]\n",
        "exs"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.0, 0.0, 0.25, 0.25, 0.5, 0.5, 0.75, 0.75]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TcumTdu5Ulsh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "08bb17e0-2631-4676-ff20-71182521a7d3"
      },
      "source": [
        "# angle 계산\n",
        "angles = [np.power(10000, ex) for ex in exs]\n",
        "angles"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1.0, 1.0, 10.0, 10.0, 100.0, 100.0, 1000.0, 1000.0]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lal4dGPqVDGW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0892d8ab-dd0b-4e59-e2de-33f3a9520158"
      },
      "source": [
        "# position angle 계산\n",
        "pos_angles = np.array([[pos / angle for angle in angles] for pos in range(config.n_seq)])\n",
        "pos_angles"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.e+00, 0.e+00, 0.e+00, 0.e+00, 0.e+00, 0.e+00, 0.e+00, 0.e+00],\n",
              "       [1.e+00, 1.e+00, 1.e-01, 1.e-01, 1.e-02, 1.e-02, 1.e-03, 1.e-03],\n",
              "       [2.e+00, 2.e+00, 2.e-01, 2.e-01, 2.e-02, 2.e-02, 2.e-03, 2.e-03],\n",
              "       [3.e+00, 3.e+00, 3.e-01, 3.e-01, 3.e-02, 3.e-02, 3.e-03, 3.e-03],\n",
              "       [4.e+00, 4.e+00, 4.e-01, 4.e-01, 4.e-02, 4.e-02, 4.e-03, 4.e-03],\n",
              "       [5.e+00, 5.e+00, 5.e-01, 5.e-01, 5.e-02, 5.e-02, 5.e-03, 5.e-03],\n",
              "       [6.e+00, 6.e+00, 6.e-01, 6.e-01, 6.e-02, 6.e-02, 6.e-03, 6.e-03],\n",
              "       [7.e+00, 7.e+00, 7.e-01, 7.e-01, 7.e-02, 7.e-02, 7.e-03, 7.e-03],\n",
              "       [8.e+00, 8.e+00, 8.e-01, 8.e-01, 8.e-02, 8.e-02, 8.e-03, 8.e-03],\n",
              "       [9.e+00, 9.e+00, 9.e-01, 9.e-01, 9.e-02, 9.e-02, 9.e-03, 9.e-03]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tJIhlEoyVU4o",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c01266ef-90a2-4b3c-d6a9-d9a0b0aa542f"
      },
      "source": [
        "# 짝수는 sin, 홀수는 cos\n",
        "pos_encoding = pos_angles\n",
        "pos_encoding[:, 0::2] = np.sin(pos_encoding[:, 0::2])\n",
        "pos_encoding[:, 1::2] = np.cos(pos_encoding[:, 1::2])\n",
        "pos_encoding"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[ 0.00000000e+00,  1.00000000e+00,  0.00000000e+00,\n",
              "         1.00000000e+00,  0.00000000e+00,  1.00000000e+00,\n",
              "         0.00000000e+00,  1.00000000e+00],\n",
              "       [ 8.41470985e-01,  5.40302306e-01,  9.98334166e-02,\n",
              "         9.95004165e-01,  9.99983333e-03,  9.99950000e-01,\n",
              "         9.99999833e-04,  9.99999500e-01],\n",
              "       [ 9.09297427e-01, -4.16146837e-01,  1.98669331e-01,\n",
              "         9.80066578e-01,  1.99986667e-02,  9.99800007e-01,\n",
              "         1.99999867e-03,  9.99998000e-01],\n",
              "       [ 1.41120008e-01, -9.89992497e-01,  2.95520207e-01,\n",
              "         9.55336489e-01,  2.99955002e-02,  9.99550034e-01,\n",
              "         2.99999550e-03,  9.99995500e-01],\n",
              "       [-7.56802495e-01, -6.53643621e-01,  3.89418342e-01,\n",
              "         9.21060994e-01,  3.99893342e-02,  9.99200107e-01,\n",
              "         3.99998933e-03,  9.99992000e-01],\n",
              "       [-9.58924275e-01,  2.83662185e-01,  4.79425539e-01,\n",
              "         8.77582562e-01,  4.99791693e-02,  9.98750260e-01,\n",
              "         4.99997917e-03,  9.99987500e-01],\n",
              "       [-2.79415498e-01,  9.60170287e-01,  5.64642473e-01,\n",
              "         8.25335615e-01,  5.99640065e-02,  9.98200540e-01,\n",
              "         5.99996400e-03,  9.99982000e-01],\n",
              "       [ 6.56986599e-01,  7.53902254e-01,  6.44217687e-01,\n",
              "         7.64842187e-01,  6.99428473e-02,  9.97551000e-01,\n",
              "         6.99994283e-03,  9.99975500e-01],\n",
              "       [ 9.89358247e-01, -1.45500034e-01,  7.17356091e-01,\n",
              "         6.96706709e-01,  7.99146940e-02,  9.96801706e-01,\n",
              "         7.99991467e-03,  9.99968000e-01],\n",
              "       [ 4.12118485e-01, -9.11130262e-01,  7.83326910e-01,\n",
              "         6.21609968e-01,  8.98785492e-02,  9.95952733e-01,\n",
              "         8.99987850e-03,  9.99959500e-01]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "29xP8DjxVkAq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dde4859e-cdc5-4251-a329-60d8206829ab"
      },
      "source": [
        "# display\n",
        "plt.pcolormesh(pos_encoding, cmap='RdBu')\n",
        "plt.xlabel('Depth')\n",
        "plt.xlim((0, config.d_model))\n",
        "plt.ylabel('Position')\n",
        "plt.colorbar()\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEKCAYAAAD+XoUoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAeD0lEQVR4nO3de5RdZZ3m8e9TFUK434KICQ1hjHhjCXYaL/TYNDfBZgJLUcM0GhUm2iOIqC0gq2WGHtdCu8dLT9Nq5KoyICK0sTsKCHhpFUyACISIxKCQCISAgsolJPXMH3sXs3OoSnZSp84+dc7zWetddfblnPdXrPA7b/32u98t20RERH8YaDqAiIjonCT9iIg+kqQfEdFHkvQjIvpIkn5ERB9J0o+I6CPjlvQlXSRptaS7Kvt2lXS9pHvLn7uMV/8REU0bKQ+2HJekf5K0XNIdkl5dOTa3zJX3SprbrpjGc6R/CXBUy74zgRtszwRuKLcjInrVJTw/D1YdDcws2zzg81AMkIFzgNcABwHntGuQPG5J3/YPgMdadh8LXFq+vhQ4brz6j4ho2ih5sOpY4Msu3AzsLGlP4I3A9bYfs/1b4Ho2/uVR26R2fMhm2MP2g+Xrh4A9RjtR0jyKbz4mT9n2T1+w974dCG90Q7+4t9H+AR6dOr3pEADYauvBpkNg36nbNR0CAHfec3/TIXDAy/ZuOgQAliz7ddMhAOCnHl1je/ctff/AjtPNuqfr9rUUqJ483/b8zehuGvBAZXtluW+0/WPW6aT/HNuWNOoaEOV/uPkAe710f5/+pX/tWGwjeeqNbfmSHZOLj/9k0yEAMH3mbk2HwBXvmdV0CAD8ySGnNh0CP/rRvzQdAgA7ve6/Nx0CAM8uuXhs3z7rnmbSfrPr9vW07e74x1hTp2fvPFz+6UL5c3WH+4+I2DgJDQzWam2wCtirsj293Dfa/jHrdNJfAAxfhZ4LfLPD/UdEbIIYmDS5VmuDBcA7y1k8rwUeL0vg1wJHStqlvIB7ZLlvzMatvCPpcuAQYKqklRRXos8DrpR0EvBr4G3j1X9ExBYpR/rt+agR8+BWALa/ACwE3gQsB54E3l0ee0zS3wOLyo861/bGLgjXNm5J3/YJoxw6bLz6jIgYKwEabE/S30geHD5u4P2jHLsIuKgtgVQ0diE3IqIrSQy0aaTfjZL0IyJatKu8042S9CMiqtpY0+9GSfoRERVCDEzaqukwxk2SfkREVUb6ERH9JUk/IqJfSG2bstmNkvQjIipERvoREf1DAwy2Z4mFrpSkHxFRpYz0IyL6hsjsnYiIvpKkHxHRLzJPPyKinyTpR0T0DUkMbJXZOxER/SHlnYiI/pKk37BVDzzEWR88r9EYnrzn6kb7B/j65x9oOgQA/vC7p5sOoeMPd47+MjCgpkMYNxMi6UdEdIoklKQfEdE/Bgfb97ekpKOAzwGDwAW2z2s5/hngL8vNbYEX2N65PLYeuLM8dr/t2WONJ0k/IqJKtG2kL2kQOB84AlgJLJK0wPbdw+fYPr1y/qnAgZWPeMr2AW0JppTSaERERbHKpmq1Gg4CltteYXstcAVw7EbOPwG4fOy/xeiS9CMiNiAGVK8BUyUtrrR5LR82DajOwFhZ7nt+r9LewAzgxsruKeXn3izpuHb8dinvRERUbV55Z43tWW3qeQ5wle31lX17214laV/gRkl32v7lWDrJSD8iokUbyzurgL0q29PLfSOZQ0tpx/aq8ucK4HtsWO/fIkn6EREVEgxOUq1WwyJgpqQZkiZTJPYFz+9TLwV2AX5S2beLpK3L11OBg4G7W9+7uVLeiYhoIbVn9o7tdZJOAa6lmLJ5ke2lks4FFtse/gKYA1xh25W3vwz4oqQhigH6edVZP1sqST8iokJSW+/Itb0QWNiy7+Mt2/9jhPf9GNi/bYGUkvQjIlrkjtyIiD6SpB8R0S/E8Bz8npSkHxFRIcTApN6d2JikHxFRpSytHBHRV9o1ZbMbJelHRFQUC641HcX4aeRXk3S6pKWS7pJ0uaQpTcQREfE8ZXmnTpuIOp70JU0DPgDMsv1KirvU5nQ6joiIkYmBwYFabSJqqrwzCdhG0rMUT4r5TUNxRERsQLmQ217lMqH/CNwPPAVcZ/u61vPKdannAUze+QXMeusJnQ20xSs/fV+j/QPs9sIdmg4BgDW/eaLpEHr6Qls0r5dvzmqivLMLxZNjZgAvAraTdGLrebbn255le9ZW2+3c6TAjok9JMDigWm0iaqIodThwn+1HbD8LXA28voE4IiJG1MtJv4ma/v3AayVtS1HeOQxY3EAcERHPIyZuQq+jiZr+LZKuAm4D1gG3A/M7HUdExEgkmJxlGNrL9jnAOU30HRGxMRJMykg/IqI/CFLeiYjoG+rtmn7vFq4iIrZAMdIfqNVqfZ50lKR7JC2XdOYIx98l6RFJS8p2cuXYXEn3lm1uO36/jPQjIlq0a6QvaRA4HzgCWAkskrRghAecf832KS3v3ZXi2ucswMCt5Xt/O5aYMtKPiKgYkJg8aaBWq+EgYLntFbbXAldQ3JxaxxuB620/Vib664GjtuiXqkjSj4hoMSjVasBUSYsrbV7LR00DHqhsryz3tXqLpDskXSVpr81872ZJeSciomJ4GYaa1tieNcYuvwVcbvsZSe8FLgUOHeNnjioj/YiIFm1chmEVsFdle3q57zm2H7X9TLl5AfCndd+7JZL0IyIqhm/OqtNqWATMlDRD0mSKZ4cs2LA/7VnZnA0sK19fCxwpaZdyocojy31jkvJORESFUNuWYbC9TtIpFMl6ELjI9lJJ5wKLbS8APiBpNsWyNI8B7yrf+5ikv6f44gA41/ZjY40pST8iomIza/qbZHshsLBl38crr88CzhrlvRcBF7UtGJL0IyI2kGUYIiL6SZtH+t0mST8ioiLr6UdE9Jkk/YiIPjGQh6g0b7/t13HTf17TaAy7nPGTRvsHePl75jQdAgD3L1vZdAj08EAsmpaafkRE/xDPravTk5L0IyJaDCTpR0T0BwGDvZvzk/QjIjYgGEhNPyKiPwjYquajECeiJP2IiIqUdyIi+omU8k5ERL8Qmb0TEdFXUt6JiOgTEmw1mAu5ERF9IeWdiIg+08vlnd79GyYiYgsIMaB6rdbnSUdJukfScklnjnD8Q5LulnSHpBsk7V05tl7SkrItaH3vlshIPyKiqo2rbEoaBM4HjgBWAoskLbB9d+W024FZtp+U9DfAp4C3l8eesn1AW4IpZaQfEVFR1PTrtRoOApbbXmF7LXAFcGz1BNs32X6y3LwZmN7GX+d5kvQjIiqGl2Go04CpkhZX2ryWj5sGPFDZXlnuG81JwLcr21PKz71Z0nHt+P1S3omIqBJsxozNNbZntaVb6URgFvAXld17214laV/gRkl32v7lWPppZKQvaWdJV0n6uaRlkl7XRBwREa2Gp2y26ULuKmCvyvb0ct+GfUqHA2cDs20/M7zf9qry5wrge8CBW/yLlZoq73wO+I7tlwKvApY1FEdERIviyVl1Wg2LgJmSZkiaDMwBNpiFI+lA4IsUCX91Zf8ukrYuX08FDgaqF4C3SMfLO5J2At4AvAugvLixttNxRESMpJ03Z9leJ+kU4FpgELjI9lJJ5wKLbS8A/gHYHvi6in7vtz0beBnwRUlDFAP081pm/WyRJmr6M4BHgIslvQq4FTjN9h+rJ5UXROYB/MkeUzseZET0p2IZhvbdnWV7IbCwZd/HK68PH+V9Pwb2b1sgpSaS/iTg1cCptm+R9DngTODvqifZng/MB9hDW/uMvzij44FWXb30lkb7B7j256s3fVIH/PDxR5oOoafvmNxcspsOoef08CoMjdT0VwIrbQ9n0asovgQiIrrCAKrVJqKOJ33bDwEPSNqv3HUYbbg4ERHRDqIY6ddpE1FT8/RPBS4rr2avAN7dUBwREc/Tww/Oaibp215CcRNCRER3mcCj+DpqlXckvVnSvZIel/SEpN9LemK8g4uI6DS1d55+16k70v8U8F9s5yaqiOh5Ke/Aw0n4EdEvejjn1076iyV9DfhXoLouxNXjElVEREPyuMTCjsCTwJGVfQaS9COi5/Rwzq+X9G1nSmVE9I1eftBI3dk70yVdI2l12b4haVyf7hIR0QSVj0us0yaiul9oF1MsB/qisn2r3BcR0XN6+Y7cukl/d9sX215XtkuA3ccxroiIRogiMdZpE1HduB+VdKKkwbKdCDw6noFFRDRFUq02EdVN+u8B3gY8BDwIHE/Wy4mIXqTi5qw6bSKqO3vn18DscY4lIqJxoref17DRpC/po7Y/Jen/UMzL34DtD4xbZBERDZmopZs6NlXeGV56YTHFYw1bW0RETynuyG1feUfSUZLukbRc0pkjHN9a0tfK47dI2qdy7Kxy/z2S3tiO32+jI33b3ypfPmn76y2BvrUdAUREdJt2jfMlDQLnA0dQPDVwkaQFLQ84Pwn4re0XS5oDfBJ4u6SXA3OAV1BMlf+upJfYXj+WmOpeyD2r5r6IiAlODKheq+EgYLntFbbXAlcAx7accyxwafn6KuAwFfWlY4ErbD9j+z5gefl5Y7Kpmv7RwJuAaZL+qXJoR2DdWDuPiOg67b3xahrwQGV7JfCa0c6xvU7S48Bu5f6bW947bawBbWr2zm8o6vmz2bCG/3vg9LF2XtdOkwd54/SdO9XdiHY7/YRG+weY+fELmw4BgGf/+HjTIfT0hbZolmw0VLuCMlXS4sr2fNvzxyGsttlUTf9nwM8kXWY7I/uI6AvyUN1T19je2KNfVwF7Vbanl/tGOmelpEnAThQ3v9Z572bbaE1f0pXly9sl3VFpd0q6Y6ydR0R0H4OH6rVNWwTMlDRD0mSKC7MLWs5ZAMwtXx8P3Gjb5f455eyeGcBM4Kdj/e02Vd45rfx5zFg7ioiYMPy825K28GO8TtIpwLXAIHCR7aWSzgUW214AXAh8RdJy4DGKLwbK864E7qa4hvr+sc7cgU2Xdx4sX64BnrI9JOklwEuBb4+184iIrmPXHcXX/DgvBBa27Pt45fXTwIhT4G1/AvhE24Kh/pTNHwBTJE0DrgPeAVzSzkAiIrqFPFSrTUR1k75sPwm8GfgX22+luGEgIqLHGIbW1WsTUN1n5ErS64C/prh7DIr6VEREbzFtLe90m7pJ/4MUd+BeU15c2Be4afzCiohoimGoz5O+7e8D35e0vaTtba8AssJmRPSkiVqvr6Pug9H3l3Q7sBS4W9KtklLTj4je1L55+l2nbnnni8CHbN8EIOkQ4EvA68cproiIZthQfxmGCadu0t9uOOED2P6epO3GKaaIiEb1cnmnbtJfIenvgK+U2ycCK8YnpIiIJrX35qxuszkPRt8duBr4BjC13BcR0Xv6taYvaQrwPuDFwJ3Ah20/24nAIiIa0eZlGLrNpso7lwLPAj8EjgZeRjFnPyKiJ4n+rum/3Pb+AJIupA3LekZEdDfD+t6dvbOpmv5zpZx2P0RF0qCk2yX9Wzs/NyJiTIaXYejHmj7wKklPlK8FbFNuC7DtHcfQ92nAMorn7UZEdI1eLu9sdKRve9D2jmXbwfakyustTtaSpgN/BVywpZ8RETE+2vrkrK5Td55+u30W+Ciww2gnSJoHzAN40fS9eMktP+hQaCP71O77N9o/wDH/e5umQwBg7ZNPbPqkcTaQ56LHeJqgCb2OuvP020bSMcBq27du7Dzb823Psj1r192mdii6iOh7w8sw1GkTUBMj/YOB2ZLeBEwBdpT0VdsnNhBLREQL43W9eztSx0f6ts+yPd32PhQPAL4xCT8iuobpyEhf0q6Srpd0b/lzlxHOOUDSTyQtlXSHpLdXjl0i6T5JS8p2QJ1+O570IyK6mTFev75WG6MzgRtszwRuKLdbPQm80/YrgKOAz0rauXL8b20fULYldTptNOnb/p7tY5qMISJiA6Z4cladNjbHUqx6QPnzuOeFYv/C9r3l698AqynWQdtiGelHRGxgsy7kTpW0uNLmbUZHe9h+sHz9ELDHxk6WdBAwGfhlZfcnyrLPZyRtXafTpqZsRkR0J2/Whdw1tmeNdlDSd4EXjnDo7A27tCV5I5+zJ8XS9nPt5+aTnkXxZTEZmA+cAZy7qYCT9CMiNmDcpumYtg8f7ZikhyXtafvBMqmvHuW8HYF/B862fXPls4f/SnhG0sXAR+rElPJORERVh2bvAAuAueXrucA3W0+QNBm4Bviy7ataju1Z/hTF9YC76nSapB8RsQF36kLuecARku4FDi+3kTRL0vASNW8D3gC8a4SpmZdJupPiWSdTgf9Vp9OUdyIiqkw7pmNuuhv7UeCwEfYvBk4uX38V+Ooo7z90S/pN0o+I2IAn7BILdSTpR0RUbd7snQknST8iYgMZ6UdE9I/h2Ts9Kkk/IqLCGI99Zk7XStKPiKjKSD8ioo/Y+Nm1TUcxbpL0IyI24HbceNW1kvQjIlqlvBMR0SfcvgXXutGESPrL7nuYWSd+ptEY7rug+Sc63r/zlKZDAGD9M081HQKDUtMhRA/L7J2IiH5h4/VJ+hERfcE2Q8+uazqMcZOkHxFRZTLSj4joJ0n6ERF9wjZDHVhPvylJ+hERLTJ7JyKiX2T2TkRE/+jU7B1JuwJfA/YBfgW8zfZvRzhvPcVzcAHutz273D8DuALYDbgVeIftTS4alAejR0S0GFo/VKuN0ZnADbZnAjeU2yN5yvYBZZtd2f9J4DO2Xwz8FjipTqdJ+hERVeWUzTptjI4FLi1fXwocV/eNkgQcCly1ue9P0o+IqCpr+jWT/lRJiytt3mb0tIftB8vXDwF7jHLelPKzb5Y0nNh3A35ne7gOtRKYVqfT1PQjIirMZs3eWWN71mgHJX0XeOEIh87eoE/bkjzKx+xte5WkfYEbJd0JPF43wFZJ+hERVTZDa9tzIdf24aMdk/SwpD1tPyhpT2D1KJ+xqvy5QtL3gAOBbwA7S5pUjvanA6vqxJTyTkRElWFoaKhWG6MFwNzy9Vzgm60nSNpF0tbl66nAwcDdtg3cBBy/sfePJEk/IqLCbFZNfyzOA46QdC9weLmNpFmSLijPeRmwWNLPKJL8ebbvLo+dAXxI0nKKGv+FdTpNeSciosrgDizDYPtR4LAR9i8GTi5f/xjYf5T3rwAO2tx+k/QjIjbgnl6GoePlHUl7SbpJ0t2Slko6rdMxRESMqnPz9BvRxEh/HfBh27dJ2gG4VdL1lTpVRERjbLO+TbN3ulHHk355M8KD5evfS1pGcVNBkn5EdIHeLu80WtOXtA/FnNNbRjg2D5gHMDBlp47GFRF9LE/OGh+Stqe4weCDtp9oPW57PjAfYPJu+3jb3WrdYTxuTtn6VY32D3DelO647j60bpML+Y07qekIomcZvH60m2MnvkayiKStKBL+ZbavbiKGiIiRGLdjBc2u1fGkX64OdyGwzPanO91/RMRGGTyUkX47HQy8A7hT0pJy38dsL2wgloiIDdiwfm2ekds2tv8DSEU2IrqTnZp+REQ/GUrSj4joE5myGRHRPwwM5UJuRESfsHMhNyKiXzg3Z0VE9JEk/YiIfpI7ciMi+keP35GbZ+RGRFSYYp5+nTYWknaVdL2ke8ufu4xwzl9KWlJpT0s6rjx2iaT7KscOqNNvkn5ERJXN0Nr1tdoYnQncYHsmcEO53RKKb7J9gO0DgEOBJ4HrKqf87fBx20ta3z+SJP2IiAq7MyN94Fjg0vL1pcBxmzj/eODbtp8cS6dJ+hERLTw0VKuN0R7lkwQBHgL22MT5c4DLW/Z9QtIdkj4jaes6neZCbkRElTdrFD9V0uLK9vzyAVAASPou8MIR3nf2hl3akkbtVNKewP7AtZXdZ1F8WUymeODUGcC5mwo4ST8iomrz5umvsT1r1I+yDx/tmKSHJe1p+8Eyqa/eSD9vA66x/Wzls4f/SnhG0sXAR+oEnPJORESFKRZcq9PGaAEwt3w9F/jmRs49gZbSTvlFMfxgquOAu+p0mpF+RESVzfq1Hbk56zzgSkknAb+mGM0jaRbwPtsnl9v7AHsB3295/2WSdqd4PskS4H11Ok3Sj4iosGHI439zlu1HgcNG2L8YOLmy/Stg2gjnHbol/U6IpL//3rvyowv+a6Mx7Pj69zfaP8CX3nJ+0yF0DXXgf8roX+t7+N/XhEj6ERGdYqCH11tL0o+IaJWRfkREnxgyrO3hBdeS9CMiWqS8ExHRJ4xT3omI6Be5kBsR0WeS9CMi+oSd2TsREX3DZPZORETfSE0/IqLPpLwTEdEnipp+01GMnyT9iIgWGelHRPQJAx1ZTb8hSfoRERXGmb0TEdEvitk7SfoREf2hxy/kNvJgdElHSbpH0nJJZzYRQ0TESIZH+nXaRNTxkb6kQeB84AhgJbBI0gLbd3c6loiIkfTySL+J8s5BwHLbKwAkXQEcCyTpR0TjhujtZRjkDv+JIul44CjbJ5fb7wBeY/uUlvPmAfPKzVcCd3U00OebCqxpOAbojji6IQbojji6IQbojji6IQaA/WzvsKVvlvQdit+ljjW2j9rSvprQtRdybc8H5gNIWmx7VpPxdEMM3RJHN8TQLXF0QwzdEkc3xDAcx1jeP9GS+OZq4kLuKmCvyvb0cl9ERIyzJpL+ImCmpBmSJgNzgAUNxBER0Xc6Xt6xvU7SKcC1wCBwke2lm3jb/PGPbJO6IQbojji6IQbojji6IQbojji6IQbonji6Uscv5EZERHMauTkrIiKakaQfEdFHujrpd8NyDZIukrRaUmP3CUjaS9JNku6WtFTSaQ3FMUXSTyX9rIzjfzYRRxnLoKTbJf1bgzH8StKdkpaMdZrgGGLYWdJVkn4uaZmk1zUQw37lf4Ph9oSkDzYQx+nlv8u7JF0uaUqnY5gIuramXy7X8AsqyzUAJ3R6uQZJbwD+AHzZ9is72Xclhj2BPW3fJmkH4FbguAb+WwjYzvYfJG0F/Adwmu2bOxlHGcuHgFnAjraP6XT/ZQy/AmbZbuyGJEmXAj+0fUE5G25b279rMJ5BiinYr7H96w72O43i3+PLbT8l6Upgoe1LOhXDRNHNI/3nlmuwvRYYXq6ho2z/AHis0/22xPCg7dvK178HlgHTGojDtv9Qbm5Vto6PGiRNB/4KuKDTfXcTSTsBbwAuBLC9tsmEXzoM+GUnE37FJGAbSZOAbYHfNBBD1+vmpD8NeKCyvZIGEl23kbQPcCBwS0P9D0paAqwGrrfdRByfBT5K8w84MnCdpFvLZUM6bQbwCHBxWeq6QNJ2DcRRNQe4vNOd2l4F/CNwP/Ag8Ljt6zodx0TQzUk/WkjaHvgG8EHbTzQRg+31tg+guJP6IEkdLXlJOgZYbfvWTvY7ij+3/WrgaOD9ZSmwkyYBrwY+b/tA4I9AY0uVl+Wl2cDXG+h7F4pKwAzgRcB2kk7sdBwTQTcn/SzXUFHW0L8BXGb76qbjKcsINwGdXqfkYGB2WU+/AjhU0lc7HAPw3OgS26uBayhKkp20ElhZ+WvrKoovgaYcDdxm++EG+j4cuM/2I7afBa4GXt9AHF2vm5N+lmsolRdQLwSW2f50g3HsLmnn8vU2FBfZf97JGGyfZXu67X0o/k3caLvjIzpJ25UX1SlLKkfS4ZVgbT8EPCBpv3LXYTS7RPkJNFDaKd0PvFbStuX/L4dRXPuKFt28yuaWLNfQdpIuBw4BpkpaCZxj+8IOh3Ew8A7gzrKeDvAx2ws7HMeewKXlDI0B4ErbjU2ZbNgewDVFfmES8H9tf6eBOE4FLisHRiuAdzcQw/AX3xHAe5vo3/Ytkq4CbgPWAbeT5RhG1LVTNiMiov26ubwTERFtlqQfEdFHkvQjIvpIkn5ERB9J0o+I6CNJ+tE4SevL1RmXlit4fljSFv/blPSxyut9mlwhNaLbJOlHN3jK9gG2X0Ex1/to4JwxfN7HNn1KRH9K0o+uUi5pMA84RYVBSf8gaZGkOyS9F0DSIZJ+IOnfy2cufEHSgKTzKFZaXCLpsvJjByV9qfxL4rrybuKIvpSkH13H9gqKu7BfAJxEsWLinwF/Bvw3STPKUw+iuCP15cB/At5s+0z+/18Of12eNxM4v/xL4nfAWzr320R0lyT96HZHAu8sl5+4BdiNIokD/LR83sJ6ijVf/nyUz7jP9vDyFbcC+4xjvBFdrWvX3on+JWlfYD3Fmv0CTrV9bcs5h/D8B7iMtqbIM5XX64GUd6JvZaQfXUXS7sAXgH92sTDUtcDflEtLI+kllQeFHFSuwjoAvJ3icXkAzw6fHxEbykg/usE2ZflmK4oVEr8CDC8hfQFFOea2csncR4DjymOLgH8GXkyxtv815f75wB2SbgPO7sQvEDFRZJXNmJDK8s5HmnooesRElfJOREQfyUg/IqKPZKQfEdFHkvQjIvpIkn5ERB9J0o+I6CNJ+hERfeT/AfkQMqg9BT3bAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KlbxVAZapnGy",
        "colab_type": "text"
      },
      "source": [
        "### Tutorial Position lookup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b5l6DZp9Zpuq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# position embedding 초기 화, trainable=False인 경우는 학습 되지 않음\n",
        "pos_embedding = tf.keras.layers.Embedding(config.n_seq, config.d_model, trainable=False, weights=[pos_encoding])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P4r3rU4caAdB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a654a035-212b-4d61-b300-7e56dd5211f2"
      },
      "source": [
        "# 모두 1인 값\n",
        "pos_val_1 = tf.ones_like(enc_tokens)\n",
        "pos_val_1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6), dtype=int64, numpy=array([[1, 1, 1, 1, 1, 1]])>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-z6mff4QaH5I",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ddcee995-646c-497a-f61d-4e1726b33c4c"
      },
      "source": [
        "# 0부터 시작하는 누적 합, exclusive=False인 경우는 1부터 시작\n",
        "pos_val_2 = tf.math.cumsum(pos_val_1, axis=-1, exclusive=True)\n",
        "pos_val_2 = tf.cast(pos_val_2, tf.int32)\n",
        "pos_val_2"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6), dtype=int32, numpy=array([[0, 1, 2, 3, 4, 5]], dtype=int32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5myVXp9uaVVo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2736c110-e821-4e0c-f484-2b43559caace"
      },
      "source": [
        "# embedding lookup\n",
        "pos_val = pos_embedding(pos_val_2)\n",
        "pos_val"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 8), dtype=float32, numpy=\n",
              "array([[[ 0.0000000e+00,  1.0000000e+00,  0.0000000e+00,  1.0000000e+00,\n",
              "          0.0000000e+00,  1.0000000e+00,  0.0000000e+00,  1.0000000e+00],\n",
              "        [ 8.4147096e-01,  5.4030228e-01,  9.9833414e-02,  9.9500418e-01,\n",
              "          9.9998331e-03,  9.9994999e-01,  9.9999981e-04,  9.9999952e-01],\n",
              "        [ 9.0929741e-01, -4.1614684e-01,  1.9866933e-01,  9.8006660e-01,\n",
              "          1.9998666e-02,  9.9980003e-01,  1.9999987e-03,  9.9999797e-01],\n",
              "        [ 1.4112000e-01, -9.8999250e-01,  2.9552022e-01,  9.5533651e-01,\n",
              "          2.9995501e-02,  9.9955004e-01,  2.9999956e-03,  9.9999553e-01],\n",
              "        [-7.5680250e-01, -6.5364361e-01,  3.8941833e-01,  9.2106098e-01,\n",
              "          3.9989334e-02,  9.9920011e-01,  3.9999895e-03,  9.9999201e-01],\n",
              "        [-9.5892429e-01,  2.8366220e-01,  4.7942555e-01,  8.7758255e-01,\n",
              "          4.9979169e-02,  9.9875027e-01,  4.9999794e-03,  9.9998748e-01]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6sp4Wjgwai_L",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "36ef4c0b-9109-455b-9264-43a8ae05a20c"
      },
      "source": [
        "# embedding lookup 대신 pos_encoding table에서 gather를 통해 직접 조회\n",
        "pos_val_g = tf.gather(pos_encoding, pos_val_2)\n",
        "assert np.array_equal(pos_val, tf.cast(pos_val_g, pos_val.dtype))\n",
        "pos_val_g"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 8), dtype=float64, numpy=\n",
              "array([[[ 0.00000000e+00,  1.00000000e+00,  0.00000000e+00,\n",
              "          1.00000000e+00,  0.00000000e+00,  1.00000000e+00,\n",
              "          0.00000000e+00,  1.00000000e+00],\n",
              "        [ 8.41470985e-01,  5.40302306e-01,  9.98334166e-02,\n",
              "          9.95004165e-01,  9.99983333e-03,  9.99950000e-01,\n",
              "          9.99999833e-04,  9.99999500e-01],\n",
              "        [ 9.09297427e-01, -4.16146837e-01,  1.98669331e-01,\n",
              "          9.80066578e-01,  1.99986667e-02,  9.99800007e-01,\n",
              "          1.99999867e-03,  9.99998000e-01],\n",
              "        [ 1.41120008e-01, -9.89992497e-01,  2.95520207e-01,\n",
              "          9.55336489e-01,  2.99955002e-02,  9.99550034e-01,\n",
              "          2.99999550e-03,  9.99995500e-01],\n",
              "        [-7.56802495e-01, -6.53643621e-01,  3.89418342e-01,\n",
              "          9.21060994e-01,  3.99893342e-02,  9.99200107e-01,\n",
              "          3.99998933e-03,  9.99992000e-01],\n",
              "        [-9.58924275e-01,  2.83662185e-01,  4.79425539e-01,\n",
              "          8.77582562e-01,  4.99791693e-02,  9.98750260e-01,\n",
              "          4.99997917e-03,  9.99987500e-01]]])>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rRIB0gjRJjAU",
        "colab_type": "text"
      },
      "source": [
        "### Class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RxvXF6OrbvaU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class PositionalEmbedding(tf.keras.layers.Layer):\n",
        "    \"\"\"\n",
        "    Positional Embedding Class\n",
        "    \"\"\"\n",
        "    def __init__(self, config, name=\"position_embedding\"):\n",
        "        \"\"\"\n",
        "        생성자\n",
        "        :param config: Config 객체\n",
        "        :param name: layer name\n",
        "        \"\"\"\n",
        "        super().__init__(name=name)\n",
        "        \n",
        "        pos_encoding = PositionalEmbedding.get_sinusoid_encoding(config.n_seq, config.d_model)\n",
        "        self.embedding = tf.keras.layers.Embedding(config.n_seq, config.d_model, trainable=False, weights=[pos_encoding])\n",
        "\n",
        "    def call(self, inputs):\n",
        "        \"\"\"\n",
        "        layer 실행\n",
        "        :param inputs: 입력\n",
        "        :return embed: positional embedding lookup 결과\n",
        "        \"\"\"\n",
        "        position = tf.cast(tf.math.cumsum(tf.ones_like(inputs), axis=1, exclusive=True), tf.int32)\n",
        "        embed = self.embedding(position)\n",
        "        return embed\n",
        "\n",
        "    @staticmethod\n",
        "    def get_sinusoid_encoding(n_seq, d_model):\n",
        "        \"\"\"\n",
        "        sinusoid encoding 생성\n",
        "        :param n_seq: sequence number\n",
        "        :param n_seq: model hidden dimension\n",
        "        :return: positional encoding table\n",
        "        \"\"\"\n",
        "        angles = [np.power(10000, 2 * (i_ang // 2) / d_model) for i_ang in range(d_model)]\n",
        "        pos_encoding = np.array([[pos / angle for angle in angles] for pos in range(n_seq)])\n",
        "        pos_encoding[:, 0::2] = np.sin(pos_encoding[:, 0::2])\n",
        "        pos_encoding[:, 1::2] = np.cos(pos_encoding[:, 1::2])\n",
        "        return tf.cast(pos_encoding, tf.float32)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "69ZQF9_QJtgD",
        "colab_type": "text"
      },
      "source": [
        "### Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1b-r4NFWb_1-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e850d700-8260-426a-eb74-f37b12b546e8"
      },
      "source": [
        "# PositionalEmbedding 클래스 시험\n",
        "pos_embedding = PositionalEmbedding(config)\n",
        "\n",
        "enc_pos = pos_embedding(enc_tokens)\n",
        "assert enc_pos.shape == (enc_tokens.shape[0], enc_tokens.shape[1], config.d_model)\n",
        "enc_pos"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 8), dtype=float32, numpy=\n",
              "array([[[ 0.0000000e+00,  1.0000000e+00,  0.0000000e+00,  1.0000000e+00,\n",
              "          0.0000000e+00,  1.0000000e+00,  0.0000000e+00,  1.0000000e+00],\n",
              "        [ 8.4147096e-01,  5.4030228e-01,  9.9833414e-02,  9.9500418e-01,\n",
              "          9.9998331e-03,  9.9994999e-01,  9.9999981e-04,  9.9999952e-01],\n",
              "        [ 9.0929741e-01, -4.1614684e-01,  1.9866933e-01,  9.8006660e-01,\n",
              "          1.9998666e-02,  9.9980003e-01,  1.9999987e-03,  9.9999797e-01],\n",
              "        [ 1.4112000e-01, -9.8999250e-01,  2.9552022e-01,  9.5533651e-01,\n",
              "          2.9995501e-02,  9.9955004e-01,  2.9999956e-03,  9.9999553e-01],\n",
              "        [-7.5680250e-01, -6.5364361e-01,  3.8941833e-01,  9.2106098e-01,\n",
              "          3.9989334e-02,  9.9920011e-01,  3.9999895e-03,  9.9999201e-01],\n",
              "        [-9.5892429e-01,  2.8366220e-01,  4.7942555e-01,  8.7758255e-01,\n",
              "          4.9979169e-02,  9.9875027e-01,  4.9999794e-03,  9.9998748e-01]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_730DE5u4C-o",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c6ffc711-1779-484f-9a86-b076f5532a3a"
      },
      "source": [
        "dec_pos = pos_embedding(dec_tokens)\n",
        "assert dec_pos.shape == (dec_tokens.shape[0], dec_tokens.shape[1], config.d_model)\n",
        "dec_pos"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 8), dtype=float32, numpy=\n",
              "array([[[ 0.0000000e+00,  1.0000000e+00,  0.0000000e+00,  1.0000000e+00,\n",
              "          0.0000000e+00,  1.0000000e+00,  0.0000000e+00,  1.0000000e+00],\n",
              "        [ 8.4147096e-01,  5.4030228e-01,  9.9833414e-02,  9.9500418e-01,\n",
              "          9.9998331e-03,  9.9994999e-01,  9.9999981e-04,  9.9999952e-01],\n",
              "        [ 9.0929741e-01, -4.1614684e-01,  1.9866933e-01,  9.8006660e-01,\n",
              "          1.9998666e-02,  9.9980003e-01,  1.9999987e-03,  9.9999797e-01],\n",
              "        [ 1.4112000e-01, -9.8999250e-01,  2.9552022e-01,  9.5533651e-01,\n",
              "          2.9995501e-02,  9.9955004e-01,  2.9999956e-03,  9.9999553e-01],\n",
              "        [-7.5680250e-01, -6.5364361e-01,  3.8941833e-01,  9.2106098e-01,\n",
              "          3.9989334e-02,  9.9920011e-01,  3.9999895e-03,  9.9999201e-01]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F5vdTy5bcz1t",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "993ef330-2303-4ef2-e41a-aebc82e2df6c"
      },
      "source": [
        "# 128x128 position encoding table 생성\n",
        "pos_encoding = PositionalEmbedding.get_sinusoid_encoding(128, 128)\n",
        "# display\n",
        "plt.pcolormesh(pos_encoding, cmap='RdBu')\n",
        "plt.xlabel('Depth')\n",
        "plt.xlim((0, 128))\n",
        "plt.ylabel('Position')\n",
        "plt.colorbar()\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEKCAYAAADw2zkCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOydd3hcxdXGf6MuS7IsS+69YRubbnrvJZQECCUhoRu+0BNqCIRQAgESQgvBoZheQzDdgdh0HDABA+4V9yIX2ZLVNd8f78zd3SutJdmyLZn7Po8f6d69d+7saj1zznnPeY+x1hIhQoQIEX7YSNnaE4gQIUKECFsf0WYQIUKECBGizSBChAgRIkSbQYQIESJEINoMIkSIECEC0WYQIUKECBGINoMIESJE2CowxjxmjFlujPkuyevGGHOfMWaWMeYbY8yuca+daYyZ6f6d2RLziTaDCBEiRNg6GA0ctYHXjwYGuX8jgYcAjDEdgd8DewJ7AL83xhRs6mSizSBChAgRtgKstR8CqzZwyQnAk1aYAHQwxnQDjgTetdaustauBt5lw5tKk5C2qQNsTZj0bGsy8hg8oDsA7dK0t83/ajIAJR11Prd9FgAVFTUJ99u62O99K5YBULm+GoA1PfsC0C9PYy74dhYAZd37ANBz9SIAFhf2BKDr8gUA5A/qresnz43N0/1sP3woAN8vXQdAVrt0AHpXLNfYq8oBWN5FY9RU1wLQcYXG7jagKwDfrIxVjW+fpXvqOvcCYPJMXVu4fi0AvXYdrjnU6n3NnzQdgNV5HQHYsVMqAFNL9RmlpqcGYw8uygTgqxmLAehfXab5ddNnkLlA77H3TkMAWDd1JgBp2fpaZXeWsbJk1tJgzMIdhwGwbJL+Rrb/AADafT9H5zv2dM9aAcCM2nYJ8/x6SQUAuwzqprnNXBKMPXw7fW6TZ+lv07+vPq+5C1cC0KlTBwBWrV4PQFZOBgBVlfqcU1L1lzJGP2vd5w+QkaX3VFFWBUD7fH1ea1aXauyi9vpslq8BoHs3vffFi/Ts3j2LAPh+gf7W/ft0CcaeM0+fz8B+ek8z5+o9De6v7+/02Xo/Qwf2AGDqrIUAbD9Qn9UUdzxsUOJ3oKFzw93xdzPmA7DDYH1m305v+HhHd/yNOwbYcYg7N21+g8c7ueNJTTze2R0DfB06t/HHfdzx98HYtnxlsbW2E5uAlPY9LTUVTbrWlq+cDMRfPMpaO6oZj+sBLIg7XujOJTu/STBtWY4iJaezzRz+U8a/eAsAuxRpcb0sdwcA3jzjJgD2OXR7AGbMXJlwf1V5bHN4fMo9AMz6Uv8xx9w1GoBnDsvRmL1/BMDEm/4BwB3/+h0AN53xJwCuvf8yAI545wEArhz+y2DsVLe4HDXlvwCc96dxAGw/Qv8x7596HwD/ffYbAB78jcZYuUSbxk8f/jUA179yJQD9Ho8tUhOHfwtA5UV3AzD0SF179v/eBeCe8mkApK3V+7qky0EAPH/oGQAsPl/vb7cJWqQ7dskNxn7/nP4A5Bx5MwAvL/0cgPuu/zsAAy/RGPet/BSAcbvrM+qyg/6/Db3wZABu+/GfgjHPWfg1AHd31t+o+vlXARhxwWkA/PmMOwB4YfFDABxUsrvmeYEW2w5/1PtZ/9Z1ALQ75vZg7Dn/0ec4+PgbNMY/rgXgrKsfB+DCX50AwFMvfQXAsD36ArBgzmqNlavNIS1DG8+qZaXB2P0GazGf/Pk8AI48St+p11/6GICR5x0GwIMP6P3c9NtTAbjh96N1/k/naw5X3A/Asw/9Ohj71PP1nl8f/VsAjv2Fvs8fvPAHAA74qb5rn7/6RwB2P/4aAL5+404AdjrmN5rbO3/R+zoqNva0sTo3xH0vZr6r7/nAQy8FYO44zaffIZcAMP99Hfc+SMcL3XFPdwyw5EN9P7sdcDEAS91xV3e8/CMdd96/4eMVHz8IQKf9LgJg5ScPBmMX7pt4zh+vcscdm3i85tO/AdBhn18FY1d//fiX1toRbAJS2hXZtMHHN+napjzPGNMXeMNaO7yB194A7rDWfuyO/wNcAxwEZFlrb3XnbwDKrbV3N/2d1EcUJooQIUKEpsIYTEpqk/61ABYBveKOe7pzyc5vEtp0mKhv3278cdRvOeJMWZ69dt0PgMePlEX7y1uOA+DQU28EYPKbssC89RyP3f4pC+qxfj8G4C8nyFJ+v1ghhexU7Zuzv/gSgJ1HHqzjj2cDMOTkHQEo6bEbAOW18R6Xft+9u6zuLr0Vopk/S+HCLnvIypz7D409qLfCGfMmKTRVlKE/U+ksHVeUxKz3fBc6qHWhGVsX8xoAUkuLAahrl8gv1VbJe00p6AdARqburyyvDq4xVQoL+S92ea3iarlZiV8bU6VQVZo7X+3CcSZLXkf8R1FTl3ArVe5ERooLzbhnpDrr3Nbp2KSnJ9xnU+p/df3QKW6+1W4sP//aOptwXO/+kJds62LHqSkm4bVGj03icVOwMfc0F5H1t6kwpKRlbKmHvQZcbIx5HpHFJdbaJcaYscAf40jjI4DrNvVhbXozyF46j+F3jqTz9ucCMOuD1wAoGjtGr7swQX7P7RKOC/rKI6soWRGMNWqpFuDubkEreE9u5x/mKUxxfe98AO5bOAOA3JNvAmDVk/LMev1aYY5X5ipm3DEjtuCUugUvb8kkAHYaojCKDzFk/WIvAIqrFM7Ys782izErtdn3cAv96qmKf1atjxkFGb331C9VJQmfjQt/B5tBSWbHhNdrqxX7Ti3QXDx/sb60KrgmpTK8GWhx7NAuI+EZxsVQPVdQ48JvJlNx9dq4RbbaLbB+4auqqU0Yyy/AnrsINrfwf0B3f/zCXluXuJiHF/fw6/44xS3k/tlmA4tyeLNtDP76lJTNv9A3hBTj31vD825sWhv6LH6QcJ5BywxlnkMhnyJjzEKUIZQOYK39O/AWcAwwC1gPnO1eW2WMuQX4wg11s7V2Q0R0k9CmN4MIESJE2JIwgEltmc3AWnt6I69b4KIkrz0GPNYiE3Fo05vB8lUVPPDsZKYuF0dzzlhlFOx/yXMAnPnKOwA89KUIuMf2OACA694V4fbe5GXBWHeMEgk65hhlt3x4rcaYMlhhkz2vPBKAtCcVNvpgTTYANRUiGUu2U9jo2RdEAp+enxmMPX+9LOV1H7wBwGG7i9R6btk8AOr6nALELO/hnfMAqCiRVd+xi8Itq2aJAK+tLArGTusxEABbIoLYWy0+rGWdd1GSq/flLfC6Gu8ZdAYgI1uhnjUryoKx69auShjTzy/XhZS81ZniwkTpzquqcWGilAzvGQRDUhOyzivDYaIaf2/ifziT1niYKJwK4Z/r518TChOFPYV6420oTGQaDhPZWucJtIBF3RJjRGhhGBOEIbc1tOnNIEKECBG2NFoqTNTa0KY3gx69Crj1ypP424DDAXj0TaXiFSyYqp8u7nzIfxX//7BKVuc1uSJif3H6fsFYXR9SyuiOL4hsfmyo0ibXtRdB3O4MEcxdvnoPgDvGKl8/u0C57GOmy4qfOkkW+nb79AzGbv+1PJB5Y0UQ736CLP+qMsX5l7hceo+BHZ1F7SzujoPEEy39Wjnqde1jcf2aAj3HzFLaZ2qGPJZs5wLULFPe9bpujpgNxZDrssWFtMvRcWV5ZTB2XakjmdNdPn6d5wxkpZf5OL/zjlIyUtz70vxMtieQYxa29wS8h1JV0/C8jKsZiR0negak1v/qOr44+M/qOYOUJJ5AwBGkNHzcFDSXQ/BIbcDob64n0FYch22Kd2hBzqC1oU1vBovTC7mpx5nsmPomAHcdqfzyP/3n3wCcs1iZNnefobz4s05UcdS7J1wFwM4XxDaDjv130mt2EBALW7QrVOHP6ClauPc7oC8A498WGdx12B4APD5em8aKmQoTDfzZPsHY6TnieeaN18J8BBrLLyTfLldoJj9dq1AhsVANQIeB2nC+ft8V/rSPvVaapoPsJfP0rGxlGuW4MFHVMhUwLXcLdEZKaDNwWUaFuVrQa8pjufV1ZY7cdeStzybKd5tBhV/AyzVfHyYq9dlEDYSJKkLpROVVtQ3OKzUj9NWsRyDXX7HrQoGiuiRhIE/mJnu9ofvDYaBw2KgxNGehT0r2suExthJH/YOCwZASNky2EbTpzSBChAgRtigiz6D5MMY8BhwLLPfVdcaYu4DjgCpgNnC2tXaNe+064FygFrjUWju2sWcUL1nGqFv/Suk8pWg+0FPW+Olfqvrw+b1UNVmHPIMhj4oUfqj9zgDMvuO9YKyfPfl/AFz/rKpT795JcgHv77s/AH/7l+QTnrp0XwBeuvcRAA6+7DwAxr35PwDKnHRE9iGXB2P3cT/feFVpqUfMVCVyeo5CNBPmiajtniWLI3WpQlA+n7njEJG8SytEcsd/GVe4NM6i+QvcmIUA5LsQWekipc8uWafwT3YoPlGXpTl0zFEIyhPiAHXuHm8J+TBRFzfPYk9Gr1eldLqr4PWppTatfmppZW1iWKjWeQrecq5zKa8p6fpq1vkwURPqDMJhouokhLEJmdA+jFEXSjVtDuoTzA1ftzFhpY0NRUXYPNhWN4PNWYMymvriSe8Cw621OwIzcIUSxpjtgdOAYe6evxljts1PPEKECG0XxmBSU5v0r61hs3kG1toPne5G/Ll/xx1OAE52v58APG+trQTmGmNmIWnWzzb0jLxORex7wXkMukLDzv5YGihX7XYBAE8eoUKy2fcqdfPk50Qsn9lZxOZ7y2Ox+fsOULpm4Z9HA7D3nfIUrixUgdrIy6R7M+z/xEN4a23kPqrgffn+xxPmtiArJr7VZ79jAVhc8QQAaz75EICcTvIZPpsp8vkKZ1lXTZsIQKYTk8saqArl1dU+np4djD1vtUjeTCeAlpkrzqOjI3PXL1U66ooyWfmeS/Aoq5E1XOitekdaA1SvLU14XrWz8PNc2qe37utCnEFtlUx020BstSJEINcFFceJhHFKhu71MXrC1pg7jrfSwjJbteGis3CFsU30BGqDiuX6Zn1aCwfkW2vaaFShvGEYtl3PYGtyBucAL7jfe6DNwSOpCp8xZiTS9qZXUQde7vQp3eer6vf4z44B4PwiZeeMWqtF9uP9VXk8zgmCvfyCBMFST/tjMO7ca5T7X1WmMMvcYZKlODFV6p+/dNXKS0crBOUJ5z0LVIfgM39yOqk6+J1ZMVG8U4ZpMfdhlkUfu2ynXqpNWDZfhHKnYdqQVk2SGFu7Qom5pfaW2qmvZM7IiTHIc5wCZ/f5ml92gcbIz1OdQ+kSjb3QKaL2d1k6/gu91il2dmqv62srY5tB5WqFf1LS8xPm7+sMAtK3QptBqgsfVdT6zaA+gVwZVBz7MJEL3aQmbgap6SHJi/DG0gCB7Bd7k6LX6pKFiYLMpUbqDBoQcQzmF5DQicfhkI6X00gWNopHaJ+udxyhFcCkBAkV2xq2ytfNGHM9UAM809x7rbWjrLUjrLUjitrnNn5DhAgRIrQUDFtSqG6LYot7BsaYsxCxfKiNmV4bpcK3aM5yrj3lQT5yssi7HXMFAK9+J+757A9EzJ5/40tALEXyXwUHAXD8gzEtfJ9+2tOJ3v3fcxrz+Srd28HpGX3zuEI9212iSvKacU8DsRTUwoHyGF74LKaj3reDwiw+dXSxk8nudqDSOmd+LsK4y64KLRVPVgppbmdRLjWFfYGYhZ2VH5Nkn+ZkrndarJBOTh9Z4+2K9MyyZfIclpbI4t8lPdEzKK126aKZsry9gB1A1Trdm56Vm/D8XBcmSvdhojJ5T2nZifUIpNYPE4XrDGqThYnSE4X3TMgaa1Corl5YKPH1etpFIULZOo2+hvLim51K2ozrg1BYhFYPw7abTbRFPQNjzFHA1cDx1tr1cS+9BpxmjMk0xvRDbd4+35JzixAhQoSmIPIMmokkinzXAZnAu876mmCtvdBaO9kY8yIwBYWPLrLWNmouFeRmcNquvVh5nCzoYceoenjPe9TwZeLPZR0/Ol8x+uP/T32jr7xTmkU7/nlkMFZxlaqUL/q50k5vvU0RrP9OGq9rr1ezko/e0nWXHzkYgOm/E+/QdZj0pPoPldU+ZWKsEdFrvURk7+I6a01fI+v7wO2Vvvr1m2pE0/lEyWbPeVvz7ThcFvmKqsQvVparegaYtkRW+bol8gzyCvSecxxJXjxd3MUy98wCV8zl01ZXuc5uBdnOM6iJVTdXrXVcgCOQawMCOZEzqFmvsdPb6TrvGdj0LMIIq5bWeU8hpFKaEio6C6eW1tGQaikJ5+psw5xBU632DXEKLU0oNwVhh6WxIrSG7tmmqoHjsMUI+ajOoPlIosj36Aauvw24bXPNJ0KECBE2HdFm0DrRbwA88S/GDJRq6X/fUfZN7uHyEF4YpQSlY/6ktM+nTlBMPvde8QNnj4qlf94+TBb9fjtIN+hapyg63ql43n68rPZHL5Hl+qNeslSf/UgewF6niPI4YqgKxC58fkww9hffyjM4dYA4gjcmSavouIHK/PmTUyfN2uloAJYuGQ1AX9dDYf7axIKx3A45wdhrXPOdNc7yLyqUdZ7bTe9j7ufqX+z7FGQV+F7H8gyK1+t8N5d95Iu+IMYZpGZ6z0Dncxxn4D2D6jLxEalZiZyBTa2fdeFTS3180nMGXqW0LpRNZOucVlKIMwiniQJU1yVKXVTVJh7X1iWmjtrAk2i86CxZtlD49eA4ZKmGLdfmchARWgeMMYFW17aGNr0ZzJy7lCN/eTsr3pJE9ZvDJTN9mOtT/Mmv1Gv4xcOVavrVz38OwICD1K/427dfD8ba/1GNsfRuNQwq2k5NbWq/UZhotyqRvP924ZSqN5RiOqlEi/DIfVQzMNAttvGNc5a6huV9D1WTndIvRVzv0Fnz8nLS1d1FUs93oZsR/VRnMK3Yaxdpwcwvignb+T69K1yKaP9OTpuom1JkVzntn/JSt6H4zcDpBhW7Zw0p0n3xi12l24TSO+nL7xfgrDQtZLHNwIWJcjRmsBmkxWS8PSrqpZb6MFG4zsB/NTWHcGppQxGcZGGhZJLVQZ1BkhBDojZRIr3W2OawMQjPo9nCdS02kwhJEYWJIkSIECECREVnrRLp2bl0Gb4PR02WFbyLC5m8tres5WmnK7Tzzj7yCMYuVhrmC48cCMDep4wPxhqbq/aRix+8GoBjH5au0QHvywqfeeedOt5JYaCvHlD6qm/4MiLX9QFeMRMgoU9qiZPU7nqmmuuk3vUBAPlrpHTqv1wLymV9Fjtr/qjuChO9/p1SUYe4Yq9OcZ7BgmmueY1LEe3viOPcHp0Szpevk/eR00X3ppYr9FPsPIZ26fUtXe8ZZPR0lcXOUM501VAuG5Qa5xlkdJUn4z2Dmgas93oqpd46r1d0FkpLDaWSNtTPuDqUS5rMMwiHhTxsEu2ipmBzeArJ0JLP2Bp8clM+3tbs5WytFqabG1GNY4QIESI0EcYYTErT/jVhrKOMMdONMbOMMdc28Po9xpiv3b8Zxpg1ca/Vxr32Wku8tzbtGQzr3o4JN+1KzpE3A/DPv0rq6L79LgbgZwulQPpAB3kIA1xqZ+83VFi287HHBWNdNUpKogc6IvaOoxXfL/+Veh68+VdZ86ffq8b3D/6fUk+7+1aP/3kKgLXLRAbndu0bjL12odRKU3dRE55OmbcDUDVJGkUZTr100lJ5Ll52YlBHWe+TF0lS4lDXcH5ot5gcxSer5TWsdbH4HfJ1T2ZXpZ+WORK1okxeU3ahPIf0leIIlrj3mxOK2QNUljjPIDvxa5LtOAPfWtMTyOk5iamlVeGqL2KeQb4vOnPug2+ME7SNzEgsOgsXmTUwdJwcRcMcgW97GY7Fxwhll/bagJUfPtcYAdwYYbyNGpc/CKS2gE6IE+J8EDgcye98YYx5zVo7xV9jrb0i7vpLgF3ihii31u68yROJQ5veDFZ8O4O/DTySq194FYDnOojALK/V8WG3KAz02JH9Aeh/jD7LJ678JwCjZ10XjDX86F8DcI7bMFJfVP1A5wu1Yf/vJoWFzjpeGkbzzhwNwE8GKTQy9WmR1b7/b9edfhOMvW6xwkHfo2yivq45zLKPVFeXXaiq5U/mrEp4f10zNNZqRxJ36K1NYEjXvOCaqnWr3XvWQuarndO6KFPKbyzVrqtaO9c1La3M1TCs02aQm1E/DlpdJnI5IzPxa5Ll9I18dpMnkNPaJWYdhfsdQ/3OZkGdQUaoziCkTURIBbIh3aCkYaLUDXc6awkk2xzCIZ26TQjxtJYNpJVMY+vAbFwYsQHsAcyy1s4BMMY8jwQ7pyS5/nRUq7XZEIWJIkSIEKGJkGppi4SJegAL4o43JM7ZB+gHjIs7nWWMmWiMmWCM+fEmvKUAbdozMEbW6f99/lcAtntX5Oiqt24C4A9XvALA8E/fAmBeiUjUGZfLMzj33XuCsby20I9OVGXxB7+Td1G7pyqLOzrL9Z3FTsvHhVW2P0Xexr//9knC3Eac3T34feHXrlHOXFn+A/up7mDxhLkaa7j+ll/OUohpJ/dFSl+hXs0lKxXi6ThQVr0piBHI1a5NpQ/NdMxy8tKdeiacr1rvPANH8qYvV2iq2JHEqdXx6iBClfcMQmGi1BqXSpriw0Qu/TNTnpmXum4oTLQ+RCAH2kS+ArnSewYbbmbjh47XjQ9SS1M3nFoaDgulOE/H39+QlV+vjmAzEMbh2oTGsCXI36YYwa3FY9kyMM1J+S0yxkyMOx5lrR21EQ89DXg5pMrQx1q7yBjTHxhnjPnWWjt7I8YO0KY3gwgRIkTYomhemKjYWjsiyWvNEec8Dbgo/oS1dpH7OccY8z7iE364m0HRDkM484NxXJa/KwAdTlOfgvMWyrofeqQs6QPuEPlb6doxPnKoGtK8ecXzwViH3KEq5cGHngjAg0+fAMDEV74D4PbBSl+9eayKz67qoph795/9AoDv7pAH562703frGYw9/j31M3jlf/pb37eXvgNfvqixOx8tK335QukMea+jaoZvpSmPpmC4xmzXPpa2WhvXjAagsJ1LA61RdXOQ5uk8iOxO8gwyXfFchSs6S6lYSxhV7jV/rYdxXkSa8xg8t2CyRE57IjemQxS7t9w36HHnAtLWE8jlXqU0x73um9s0nFqacC50yvMTG7oHkqcKNmT1N9UTCPcv2BjtnG1URqjNo4U4gy+AQU6YcxFa8H9W71nGDAEKiGv0ZYwpANZbayuNMUXAvsCdmzqhNr0ZfLdgLdtd/g7vXajN98qbJYc07CiRwYvfvxeAbgdcnHDfrmO18D/e5aDg3GOn7gjAP2eLkPXVvjM+Vn/lPa5Rt7Lpr0oEb6dzVJewvLsqlcuDlUg/D+gTy/jpuZ0ye+ZOVxio+4Ga77OParEf1l8L9Otfijvq4gjbtd+p73JFicJKBUP6AlDXLvZnCy9OaWuVXVSb2ynhvJemTi0cAEBWjtsMylzP4UplMsXn7Ze7EE5+u9Bm4DagNJdJVe1Ic78ZBBtQ4lqs13yDnnCYKCCQnWREZqLIXThM5IdOSagz8HITTQsThSWvw9lE8UgWJgqOm7lyN/f6jUVECrYsjIHUtE3/21lra4wxFwNjgVTgMSfYeTMw0Vrr00VPQ10g47+sQ4GHjTF16E98R3wW0saiTW8GESJEiLCl0VLKr9bat4C3QuduDB3f1MB9nwI7tMgk4tCmN4PaqnJK5k9h3G1/BmC3M0XEDjzwcgAWnPUTAPrtp37GpcXqE3z+eyJy9+kQsz5L79Y9t5QdD8CofRWSuXuFCP+UE6VFtOovqkzudpdCeA9N1Zi9XCjF9ylOm/ROMPbBOyls9fDf3wAg65JDACiuegiA/Z1g3QsrJSrX21nixd/MAaByncJa6f0Pc5NdVu+z8JZ26jrNZ3VmomdQ49pZphZ20xycd7HOtcM05QoTxXsGPi21QzuXbuv+D6RUKuTkw0Q1LvyWku3DRLrOE8jxVnB5VU3CWLVJJKzrtbkM9TxuKOSTTJsojJQQgZzsP/fGkMNBauxWYFV9KGpD825sWtuqxHVLwRizzVYgt+nNIEKECBG2NFqIM2h1aNObweB+XXnxqavZ7XjpCZ01UWT6/146CIBf58mKf2f5oQCMnyc+4IqrJWF9141HB2M9+3t5a7OHSWlzxF9V/Jf7+28AePBzKY96q3NKpqz1Z8Yrc+xmxxHMXyaF0aWvxxRRjzlvfwD+vHQeAOu7qiLax9Z36657K9fJYyl0Ute+MU1tjTgH00MeRsqaWNKB10DyBWA1i+VNrM4bAsQscK+MalxjnKwcpZoWL5ZHUFuyMmE8iPEg+c7rqXFWo6mWN5HheIeqgEBWymuMQK5PGlSGOIOYSmmiZxCWrLapSVJL4zmDJBXHjXkKxgXWN8gZhKzuILXUV0w3YlGHCeWG0NgYkdHeOhBtBhEiRIjwQ4fZgl3VtjDa9GZQM3MGy485nB1+fAsAF/ZWEde4YdITOriTy275rdpdnnG21EsvcdkwtefcGow19df/AmKW3zfdDwJg8AGSfnjsVZH1hQOVxnrnf6ROOu8rKZIOPUXncz+SDtGsN78Lxt7rdy6W7p47tViZPd6a79dBVrC33juPUJHapFc0tu2uOVV3VM+ElCmfBmOnZ7v+BU4vpWaxPoPlPVzBWMgCr2unzKQOeZrDnPXyZOpKXCw/vb5nUJjrGuF4L6NM3oTnDNYX632FOYP1oTRSqK9a6t9zWm6iJ1CfMwh7Bg1wBkFzmobbXMZUSUk4DmNLNJ5pyoLSlLaWrRHbMu9gMEGR4raGNr0ZrCmv4bVvl/PZaB3PuOAlAN7uIa2fv05/DoDLByvl9LDXtVDvfb3CRGc89VUw1qVdtai+v9cRAFz2rF67/idqOHPq+Xfo3tNOAuDjD+dpDq6/crdbfglAZgdVPfuwE8ABs1Sd7HsJvz9XIRmfQpqxRBuND9EU7TgQgPlPK0TlwxxLSrXAF34fqy3JyFVIyVdIly5QaumSob47WuIXty5H9RKd20sA0WsW1bomPb7pDcRVNTu9prV+AV+vNFQfJvJ6TDY9O+FZFSEdIoAad85XLwdhFqdF5LV7TKibVDi11DYQJgrXGSSTrPaLVbizWT1J6wZ34PsAACAASURBVNr6QnVJj5PUFTSFhA5f01TiehuNVrRumG1XwrpNbwYRIkSIsKWxrXo+bXoz6DmoG7c/8Dtu3EOpo48fL9J34o2y7s+fKDJ4n3z9fGupUiLfOEd6Ql2OvS0Y6/CHRwJwZVepwl75W7XOPOoX0hjyFbxXHS5p65888yIQC/0s7aIwUbcfyeKdfeWrwdirx8lLyOmsyuP3Jis19MI81zN4sooLM/NUfNZuqAjmZZV6hvco5jm56bRZMQI5K39vADplykJet0BjLyzRvNqHXNrSOl3X2fU8rq7Q+6pZq6rieAK5whHAuU5OOlAaXSevwhed+dRSm64xfQhnfbX3DGLPr/GhI1dx7MNEKRnyMgJrvNHUUuqhui5UdBYuKmvk2CNtM1h+rTXOvG0GPDYfJFS3tWexedCmN4MIESJE2KKIwkStEzPKMzlq8kCucq0eH3AFYh/9WEV8z12oJjZ/f1vx/i4/uQGAWeefAkDluphO1MwR1wBwZqrSO3/lCsCW/E2SH4UDJSFxSEdZ5zXOos7ppDFeniKL/Gc7yOuoios/f/+uYv9F/SWLsXCWntFjZxHFK7/4GoB2rq9B2gBJY/iir8w88QJTVuiZnebFis6yC1Swlu8s/bXfa+yFrpisv/MMvLW81qmCdnUFd7WuGK1ipbiDtOw+wdj+PeQ7DyAgo9eLQE7P0TMrnJlu09yY7q1X1ngCOfafJ9bMJjGVNDXUvyBMINdXLfV8QMxMq0smP2ESCeSUQAojpGJaT56iPmdQF04trRfvr+8NNYSG+qO0QM+UCJsdJmjRuq2hTW8GZStX8umTT7DTJGXX3D9XdQQXXPEgEAvL3Fis0M7NYxQWum4fVRsP/k1MCPCchycA8MIa6RZ1GX4UAJ+O0mawxx/P1zP/JfI5r5s0froO0cL93Hjl93fJ1QLpwzYA87/QxtLnaC3c34wXOd1j70EALJ2oezsMVOe1qk467xfVrHxVE3+zQAv2TvNKgrHz+me7+YgAX7dEG8b3K5UlNCIjMbxS4jaDIldV7DWLKte4quKMGAnsn5/viO50t6jWlolATs/R4u83DZuWSTwqa+ovjPUkq0PNbIIK5IywNlFic5uwrhDUrzMIZxPVhQhlKz6+Xgy4oWyixkJHTbUW40npCG0PJvIMIkSIECECREVnzYYx5jHgWGC5tXa4O9cReAHoC8wDTrHWrjYyze4FjgHWA2dZa//X2DO69uzK+Xdczc5nPgDA5P1FrGbmqzr4vptOBWDkZfcB0PdPalnpSdUHRu4ZjHXE6QotvT1NVvvpTyuk9MmzCqP88VjJUH919GUA9He1DXsOV0XvP59XP+NnXMjqzPyYZfut6yV8opO1/vR51TQUXbAXAN88Lq+k80GqRF64Viart6hzOonEnuF6Ia9duC4YO79IVb+53eUZLP5cXkjxas27wKV/emJ4WakIW+8ZeM2iyjUaM83VLUDyMFG1I5t9z2Ovbkp6ojXvw0fxYaJYm0v9DWp9nUG2J64dCRxOLQ2lklbXJoaEoH4P5KqacAjHj5EklXQDFchhNLcWYWNqF8Kcc2N1Bw1x1Ntq5svWIuSN2TJ1KFsDmzP4NRo4KnTuWuA/1tpBwH/cMcDRwCD3byTw0GacV4QIESJsNFJTTJP+tTVsNs/AWvuhMaZv6PQJwEHu9yeA94Fr3PknnWb3BGNMB2NMN2vtkg09o1PpMkZOuIdRWVJzve06qYK+t0CE7IAP7gdihVQ33S29oC+uPlj3z3szGCstU1bud64N5D2HixP4rfubbrdM1vvL01Qwdt5R0gnas6ca0zxyhyqOp01SG9NBB/YOxh7/sgrTTu+n1FGvQZQy/AAAZq8Wwb2bUy+dWqx4v++p0MFVUq9ZofMrXNMZgL5dVCHdvrfjI8bNA6DMvY9s5zn497fc9S/YwTXn8amdFas1dnperKWmt7SzXbMdXzFdXeb6GQScga4Pcwa+Ajk9vugsSC1Nxhk4FdUQgRxOE21I96ix/gXJ0JTmNo21uQz3J2itqaQRNg2GtrnQNwVbmjPoErfALwW6uN+TNYeutxkYY0Yi74FcUrnlprFMXadexp9MUHvRlKvUMOiv/1LF8U3vjgXgqoslP1H3sMJG43ffPxh35wv/AsDQr7VhlD50HQCHddNiO+1ONcpZ7Kptr9ihMwBZ61Tx6xfV4pnaiAaevW8wdu1L2gz6pMSIX4DirK4JY+7lNosvFyiPv8gtmIVu4Z7+P4nlraiMLUhD3Pzyems+q9zK7DeD3M5uM6jThrJ0nQjjvXtpE/OLW8Vqnc/oHFuEPYGck5a4GVStU5goq1BhLR9Oqk1JXMC99ERzCOQAqaHNoB45rJ/xYaJknc3C2UIpoeyicNgoNaVxh3lTeyA3tFkk20CSPWNjlqRoj9o0GAMZkRxFy8Jaa40xDVf9bPi+UcAogE4ms9n3R4gQIcLGwpjNU5TYGrClN4NlPvxjjOkGLHfnm9McOkDXTu34zUkjeG+AWk8eN/MjgKAnsk/v/M269wB47MiTATjxXmkF7elSUQGePFt1BLXfK9d/3O3yJg657kggpjXkx0z/8CkAKpaLsM3t0heAMlfrkHnglcHYHTOUrmq/flf35sgq/9KlgZa40Mmuzsof/ek8AM5xTW6G9dD1X737XyDWQAdgtyJZ/Dm9eya8Vr5O4Zbc7hozbaW8iyWuijnX9xx2VmflWnk2mdn1vxI+TOQJ5Kq1Cinl9ZZj5z2DqlBZsPcMcuP+88TqDBK1idKyM9185BXZ1EQCOSxZHa42hvphopqQNxFOLW0MDYWJkqGxsNA2un40CY021Nky02gRGCICuaXwGnCm+/1MYEzc+V8aYS+gpDG+IEKECBG2OEzTyOO2uGFsztTS5xBZXGSMWQj8HrgDeNEYcy7wPXCKu/wtlFY6C6WWnt2UZ6wt7MW4s/7MstFqXvOHW6Xx88hhSi3tf4x0hh4+RW0xX5z1BQA7Hv0bAE7OjsWlC1+/C4Ds390OwN0PasyfnKXKZC9x/ZNBiutPflAKqV6xs/tOGnPmeHkG8zO6B2Nv5+SZl/1H6ac5nUR4j5uxAohZvb2yZe0uW6QK38595REM7+Ga35QUA7HKZIABHR0nsL5fwmtVjqTOGSSV0oxyjbVoteL9uemJdkCl4xgys0KaQEB2wBnoZ3WZvIv0XCcRHngGiZa49wzyG0wtbYwzSCwyC3MG1WGJ0rhrTGpIwjrEEXiEU0nDjWs2hGTXhMeo20hOAVqHN9EKptCqIM+gZWxoY8xRKKU+FXjEWntH6PWzgLuIRUkesNY+4l47E/idO3+rtfaJTZ3P5swmOj3JS4c2cK0FLmrg2ggRIkRoVWgJq98Ykwo8CByOEma+MMa8Zq2dErr0BWvtxaF7OyLjegRggS/dvavZBLTpCuQF85dw2cW3s37y8wDcepKa1nd9TxlB37rCqwVXKxpV8KQKyfJ7DwXg1IMrgrHGXCVLv3qH8zSGK7R6YW5NwvHO5ylL6KXb/g3EUgoP/ZU0fRZ9I4/g9ekrgrFHDFXa57xx0wHouNePAfjvVFEmu7kvV/pipaeuWiYuoWiws+o7Kd5f5XoPxOsedfFFZZ2VyuoLwKrW69rcHpKyyCxWU5ti5wFkVJcRD+8ZZDTAGaTWuGY8zpqvLtO1xjWzqbYNewbrQ41soIFsIpcZlZoZlp9I0ubSWf1ejsKk1s8mSpZaGmgTOU/Hj9FYr4L4c8lTSxs8nRThVNSmYEtkAjVlnWsNHsvWQooxLZVNtAcwy1o7B8AY8zxKsQ9vBg3hSOBda+0qd++7qKbruU2ZUJveDLLzC9j+2BPZ6e8SbvvJZRcCsN/lLwOxdM/PrjwQgMccCXzVW28D0G33C4KxPvmzZKPfdw1vnnGhppH/0gJ9z45K3Sz8uaqYp16rDcf/pz5/Ly3G74yTUN3Ln34fjH3SoepHPPZ+EdzdTpfw3Pxp2jB8CmnFJBHbZSv0Zynatz8A7fO8jlB5vc+gY5o2q9p8pan6RbPadSPL6iaSN2umxvApp6Y8Mc21woW72uUkErcAKRWJnc082WyyQmEiH3Zxi0V5td8MYmPVugXbj2XLEyWrg8U2vBmEw0Qhsriha+o1t3GLf7JskOY0mWlsMQ8Tyk1Z/KO0z7aBZmzkRcaYiXHHo1w2JDScTr8n9XGSMeYAYAZwhbV2QZJ7ezR1UsnQpjeDCBEiRNiSaKYcRbG1dsQmPO514DlrbaUx5gJUqHvIJoy3QbTpzWBIB/js+DryLpTF/+VP9DkVPDIJiMkzl/zlrwAsu0uE8t1dlKg08rWYxs8+BQpTPDFB4Z+dH78egFkX63j3634KwMQaWdo+9OH7BA91WbLb7eD6F386Kxi787miSWbc/h8ADh6ma+79tzyF3i6FdPnEyQBUrJaV3364vBWTXpPwvuNDEqklSm0tL+ibcE2N8yJSO8lgyHbeRbnTJkpxnoHXLPLEc2FcL2L/HFOlkJJvc1nlqphTcpS2WpskTFRepXnHh4nqgjCRT21NokUULjpzQ4f7Gydck6QCOYywpHVzwkRhbGzR2ebAtlkK1frQQplCjabTW2tXxh0+AtwZd+9BoXvf39QJRd+fCBEiRGgifNFZU/41gi+AQcaYfsaYDOA0lGIf9yzTLe7weGCq+30scIQxpsAYUwAc4c5tEtq0Z7B42vfcuNdFPD1RFvZf9hI3cMs77wCx9pLH3aDjfx2rPgHvHS/l0TGDTwnGuvs2kbqXPKnUyw8yZJWXr35SPw9VFtddL30LwCkdpfUzt0w6QWVvKbPr53uqBecHz8baXjJc7ThLXBvIYweIUL7dNdDp3UMW9rL/zQegqkxjZ2yn4jm7SvyDt3gTmtwvVS+EVVnxRkbMK0rrIi4jO1fH61bJI7BrixPGLHXFYB1zYvpCPu6dUinPINzmMsVxBt5q956Bj6muD+Qo4ovOdG9adiJHUL9/QVrC/OprE9XnDCqTEMgpIQI5eEay1NINxIR9kVyYE9gaGvexlNnk3kmjBV8RUdEsGFqGQLbW1hhjLkaLeCrwmLV2sjHmZmCitfY14FJjzPFADbAKOMvdu8oYcwvaUABu9mTypqBNbwa56ans3y2X7recC8Ac98U+d4Yqfi89QfVt7R5+BIBh7z8LwMMdFMZbmzMjGGv9aRK1GzD3fQCueUZEss88+svHWpAnfjwbgDtOkqR15wnicaY8ow3p8FN+C0BFSSybaFalagF82GWHzlrsPSHc1XU8m/PePADquoi8ru6szct8JcLby0vnxn0ZqxfoPSzvtA8Q0w/y5HltrrKJOuQrjDW/TCGyWidM5wXsfBZS5/axzWC1m691mUkZLoRUtkz3mhzVP8R6HidqEZU3kE0ULLiZoWY24Z7HqWECWT+D6mJ3IqUBAjm8+PueteHjxtCcCuQwwtlF9TaPbYhQ/iFtKC0pYW2tfQvVWMWfuzHu9+uA65Lc+xjwWItMxKFNbwYRIkSIsCWxLctRtOnNIHvoEHYY9yG3Fg4H4P7PpUbq21oe1lsS1XuMlCLp6f+UVX9KJ1nq7+19bDDWGU98CcANZyg19NyLdc+uJ+iaF96YBsDyyUr/7DdqJADt+ynF1Ncd7LFI43hiFuDd2QrJdHfVvXkrZwIxK7fzCLXlfPdV56k4LdellXq9YJZSjzNzlZJakB6zhkvnKrQ0f4C8jOxQf9a6XIWkunUQWf6lq1WocxpFfp6eCO8YRyCX+V7B66QX5Ank1S40ZlyLTB8minkGuq/SeQZZcXOqq5bHkpqv5/gKXZMZa7fpJpZwGG5c00ABcj3CONAiMomqpOE2mEGYqDZ5BXI9krkRyz8ZGgrpBFXYjdT7bqNrUNtCC3oGrQ1tejOIECFChC2JqJ9BK8WUecvZ+cwHmH6/0j5/OkmFYae7NNFX56g6e9yF0gIqPE7SH88+exUAtYV7BGNdeIU4g5NOOgOAXzgL+uYfy+s4+oybdY+L8y/qqRh9r5OlVTTDNdZZ9fYrAOR1HxCM/dqXyhi7ooNrBvPVOCDW6L79LjsCsLhC+kdpWeIGphWLzB48TXxFdoGqn7tmxTyDktka+/sRmlf7ELlVamXNd3NtOH1lctXKUvcskcAVLsWzX5w20WJvUTvPINPxCZ5Args8A88Z+DaXuj/WyCbOM3BcRkqGI5BrE4vOPHxqaYwMTni5YdXSwHsI9S/wHk5tXcLrHi0pSRxY+a00jh6lD246os2gFcKkpJCelct1ndVd863bFSZ66VuFh3qdIgL500OP0w3ttJi+0UWy1Kemzw7GOrdci+OMG1Vf0H23XwKwX6oayvhNwBPKf5+g8MzIPdXxzC9EM19VmKj73ocHY8+frjBRr/0kM730Y7V3zut2EAApA3cDYtlGWfmua9lSJ1g3XaRvbi9tdh0LYyGVNXOUijzTSVjs4qqZ/SK5ukKLU/cOuqfGvc+KldoUUjO1mfnq4YI48b4sH05Zp0013WUa+d7GNr2de++6PhwmislVxxZsv1imZSXWFaQkySbyiC30TkqigWyiQI4iiTBdGNYmySZqQGQufE2MpE7cAJMhFL2rdxyhbSDFRM1tIkSIECFCxBm0TgzvW8T7o8+l034SPB3qmtfs/TcRsZ+8JQ/hslyFiY54VOGkK/6sSuCus0YFYw0+VBlcb9wn8vmC19XkZt49ErfrvL28j55Dlc//6nuqMM51ufd928nS/e4bpXDueVFMwvqfozWPPj8WOT35SaWhFh2gmpKS3ERZkZxOqg2YOFepw7vNUZimw64K6bTv2T64du1CEcPfu77JR7jmO54YLnb9kjvneH0jEcflK+V1ZLSTtHVDnoFPCa1Z67SJchLTUG16Ys9jn+cf3Oc9hTjCOwgTpYdSS+sJ1SVKWIclqzekTRQmiIOwkWsd3Zjn0KJho9rkNQCNoTFCORyJ+iGleG4tGMxGiQy2BbTpzSBChAgRtjRaKx+0qWjTm8Hab6fyTr/dOOaPqr148WSpfOYefC0ABzqL9NbtRdQecJJI3dz7Hwbg5W8XBmP9/Zm9ABh9uyzXe3eTfPRzJ0vn6ND7r9bPIYrbX/RrEc7/cqTq3UN0/RuTVPV82q4xa//Ju1SY1v6gXwDw/fWqTh58nriB6St9Wqi+ZPldnOT1fMX1VyxVnL+ba2HZoV9BMPYC54msXqUx8gpdsxuXqrlorTyBbnmO/HWVyZ4z8IVs3vBunxn7Svg01ap1TpsoT2P7NFTPGXh4zsBHVL1nkBYni10X4gxsnVRUSUvkEGpCln+YMA632ASorUskiG1QqNZwaqlHU5rb108t3XAR2caEErbRNaZF0FoWYEPz5crbCtr0ZhAhQoQIWxRm60iPbAm06c2gotYyfV0Vz+aNB+CdoWpMs/91DwHw0eOSpTjw/X8C8N2ZPwOg/wHiGHKnfhCMtfN0pXXu6tI/i+8Rh/CFa5Bz4xGShujeTnbvuU5uYv43Uhrd/jSlqT7/pRrp7NM9ZjX7OHlZT2kNzXLKoYdvr+qyLxbJSi/K0J+jsJus9cUuNXapywga3kPx/fb9YvpVKypVkLbOewbddW+aEb+wpFSW97DOuQlzKXdpq5m95DF4ziAnrh2m91QqV8szyWif6BnUpSfG+ctCKqW+d0Fqev3U0tQgm8j1VwjxD8naXNZvXJO8n0GQWhriCMKeQriNYVM8heaiOXIUyZ67MUtQKzGotxkYIL2F2l62NrTpzaDbsP5c/9rTXDrgRCC2CP37CL0+eJqqh494Zh4AO7+hyt9n5krQLmP54GCs8SOVlnrUhXsD8M692ii8DlDPmaowrlk8F4DsAslMl8yXkGDhXyRQl/9bVSSnfPVmMLavG/hisRbVZZVaNE/urXDPDW9pjJNche8OfV3twgQ11il2i+zw7iKO80v7BGOvcGOVuXBQnttI0tdp45jvFv38oQpv+YWmfLWuzxqSGJ5p18BmULXO9U3uqXBbsp7Hpa5BTrbxBHJiIxuIq/LNSBSqS9rZLFmYqKZ+nUFMqK5hArmpaEqns8bCFtuo8dgkNCqOt2WmsVkQhYkiRIgQIQIYE4WJWiMmr6hm54eWMOogWcoDj1MY5oE9FC56c5YUXnc8+jcAHOnI0UHvSXco+577g7Ee7KYGNIdefTsAX9whQvkng2Slf/vHvwNQ46zfXrtqzJnjFRb6Pk9hpKGOqF36xtvB2Lldldr6hpPU9lbvgFz9Mm+uwkE9+6tP8a599PMpJ3Hti9GGFCn0k16xXTD2WmcNV7qwVd4geQAZc+UZfL9S5G9eRqJrW+E9g3aJlb+JYSJHIK/VGOm5iW0uvSXu4VVK871l7sNEDRSdpYaKzsIVyMnCROHX43sgB6mlSVJHk0pWbwSBHBszeaFac9Ea1phWMIVWDUPrIbNbGm16M4gQIUKELY0oTNQKUVm6ltkfjaXmGekBjSlWTL6kWnH7tNukLOoLxs45XQTok5e/oOt6nxGMNcAVZT3wnazgvs5i3uuaYwAYddmLQOyLcNKVAwF4cLrSVZ/+Slb8kbuKS5j15uRg7M5HnATAJ5MkK7GPGyRtnnplr1wiArnLTro3p4u4gcp1Kjrzlnj3XM0ppWu/YGxfAOavzestUjprlfiIZY4Az6iKtfgEqFwr4jYrJ9EiT61eH/ye7Sz6qnXyInz/gmpHzFaErPV1zmvqHdICSs+KSy115HlqZk7CvWHOoNqnhTrL3zez98cNcQa14SIz7yk43qcuJD8RvOeNSTVt5oKwMYVKW8IAbcwbaQ3eSmuCMZC+jWqJtOnNoF+frtw56lqOOev2hPPFr0tf6NqjJC730uy/AbA6W0Tz1KsV2nn1kfeDez67RLpFBz/zNQBjfqxQTNZp1wAw+7ynE55xwx6qRB6zg0TmxnwoYvmiE9Q454nrxgTXDnZ1Dl98qMpoL2Vd+l9lQa1brJqALj8eAkBRByfv7DJvPDpYbVQ1BbGuZn49rl6vKuGsnn0ByJ6lcFWZW/RTyqRh5BfPMic21z4vMYsnpWJt8HuG23wqS9wY7VTnECaQg2Y21b6ZjZtbTQME8lq3iLuK44BATk0MG9UGInOeQG56NlFAILsx0kKbQ2NhogYJ5EYJ4w3XIYTREgv9trkktW5EYaIIESJEiABEYaIWhTHmCuA8wALfAmcD3YDngULgS+AX1tqqpIMAWUvmMfCWc+m6g8JBZcVLAfhV8c4AjMiV1dv7KXkKR5dKvfSh/aX9849pE4KxOj2u2oSF+14CwNCXbwFg1LeyqHs5zZ7VzvrtPEv6Rkfvq3DRI6OUSlrwF+kfzbvs5WDsH++sauT3npa3MDhPVvDij78BoHx1XwBydjlMN1QUJ7xPnzKbtnIeAMVFw+p9FtVOjTStm0JIOe31p/X1ByllCiN5S7rEv4/2stD9FzylvCQYM92FzqpcM5sUFyaKEciJPY9LK6oT5tuQNlGynsfUk6wOE8gNp5bGI2nbyySW3MbUEYRVS7ckGut53JQpRfpFmwaDaTHPwBhzFHAv6oH8iLX2jtDrv0brZA2wAjjHWvu9e60WrZ0A8621x2/qfLa4p2mM6QFcCoyw1g5HH8RpwJ+Ae6y1A4HVwLlbem4RIkSIsEE41dKm/NvgMMakAg8CRwPbA6cbY7YPXfYVWid3BF4G7ox7rdxau7P7t8kbAWy9MFEakG2MqQbaAUuAQ4CfudefAG4CHtrQIMtXV/DQS1OZskRx+zkpSg/1qaQPvX4TADccqib1U4cp9r3726pM7nTFf4KxRr6sAq+MHKVkvlamKt+/vaLeA6P2kgrp0imy2mc/LD2kM297EoC/3Oga0xf9CohZzwAHuFTR8tXyXLrvIpJ34QQ1pqnJkhYRfeXRmIWqKk51zWN84VvVbBkCK9oNCcb2VnjAL3QSn5DbQemqy+ZL8bRmpZ6d6jSLSp1V39lxBmXe2imPcQaZ7eUZ+DTUlBxxBj6eX1GTqOW/3qWW+pTUWLVxjKQOrNqQFpFNbazoLNFTCGsXAdQEpLJ/lm3wOJnOUD0OIU5xtDFrsLHQQVOsybZitP+QvQtxBi0y1B7ALGvtHABjzPPACcAUf4G1dnzc9ROAM9iM2OKbgbV2kTHmbmA+UA78G4WF1lhra9xlC4EeDd1vjBkJjATo2bE9t996GvcPOhqAgw/tC0A/J2l9V5Ua0QQyDwNVh3DRh8qsuWTkwcG4d92lbKHtDlSo5g/PfgXAvAn6e+zyOzW7WfLv9wH4eLQygX72++8T5vfubC3CXeMyaLpXL0u4pvs+Iqe/+NT9rV1yUHG6xO7yZ+jZvslNkcvqKZ+tCurZPeMyflITF7La9trEenbUxjfVZRnVrlQIJy0jUYa6q5PfWOA3lZKVwdhZToTPy2SbbL8Z6PVSVxntF1NfZxBsUK7fcVpebOGvSxomaliozsPXGaSEwkTx4ZqW7HncGBqvQG44pNPUfscao9nTirCZ0Uw5iiJjzMS441HWWq+b3wNYEPfaQmDPDYx1LvB23HGWG7sGuMNa+2pTJ5UMW3wzMMYUoB2wH7AGeAk4qqn3uw9zFMAufbptuJVVhAgRIrQkTLO61BVba0ds8iONOQMYARwYd7qPM6z7A+OMMd9aa2c3PELTsDXCRIcBc621KwCMMa8A+wIdjDFpzjvoCSxqbKCFaQVcU3gae6ZrU3x4jEI1X5WrrWXXY5RauvRp6Qa17yYC+Zrr/wFA8VNnBmPduESf4x/POB+AE89Vumq164W8cvfTAOjTVyGae+6Rx3bMGIWJfM/jpyfIU7gwP5ayWTPxHSDW87hwP4W1FtwivSOvXTTV6QgN/Frvo12hdJK8l7Fq2jwA5u4Y8wzapyU2gSkzsrh7FsgDqCyVZ1C9QvekOou8zHkGPbJlkS/ztQFxnkG6q0GoLpVXUedqA2I9j32bS/c+3XG6a7ATCxPFPgtvKEMQ3AAAIABJREFUjddrZpOk53EybaJwv2OICdM1hsaa1zScWtrwtVHP4x8WWjC1dBHQK+64wTXPGHMYcD1woLW20p+31i5yP+cYY94HdgE2aTPYGt+V+cBexph2Rv78oShONh442V1zJjAmyf0RIkSIsJWgTmdN+dcIvgAGGWP6GWMyUBLNawlPMmYX4GHgeGvt8rjzBcaYTPd7ETKmp7CJ2BqcwX+NMS8D/0Pxrq9Q2OdN4HljzK3u3KONjbVyyXJG334f98//LwA7n6695MMd9wHA9Fb18DPdVWx2jlGjmivKRJJOvuzSYKw++yg99fDUOQDUuiYwHfur/eXt47TpXra/rHpvmU555lMAeh3yOwBmfyd+YODBMWXRxe+qzWV+L0XDUoZqfqs84VogQvmz78U35H+npjt5/XW+c2dZ5Cun6/sweVGM5B3hrHBvMa8s15i9OqrausalnJYt1dgZOeIrPMFd5Cqts3xcfd3qYOzM9vIuympcYZhrZuM5g3VV3jPQvdWVzkp2HIcNNbKJR0pGuM1lmED2lr/slbCEdVCBHPefzoY4gtraULObUAVyMl2hhiqQY+mqiaR5MoRDCdto0eoPDi3lGVhra4wxFwNjUUblY9baycaYm4GJ1trXgLuAXOAl9z33KaRDgYeNMXXIoL/DWtv2NgMAa+3vgd+HTs9BDHuT0b5zJw66aCSDLtaG+u6j6lvwWFct2Nc+rDDQNTeperjr188DsOfVDwLwwjVnB2Pd9oW4m2/cuT57KbTUd6hCO2+8PTXh2cNdfv4X30kg7kc39gXggT8/A8CACw4Krp1wi2oOup8sTnx5RueEsXKdvMQnM5WpNGK6QjWF+4qwLXACdsXu/OylMWmJY10IyWceLV4nT7KHm191haqWy5drkU8P9Twucr2bfQZQ9Zo1wdiZHfT8oLOZe4ZHrOLY1RW4zcHLT9TvXZC853GdSdzUwot/IEcRqkNokEBuAz2PG1pPop7HMbTWsJvkKFpmbtbat4C3QudujPv9sCT3fQrs0CKTiENUgRwhQoQIzUAr3ac2GW16M+ifuo5n88bTY7Hy4A++RFb5f3+reoPO5nMAblkri/uNJbKo37xADsj1V8aqWE9KkeV/21sKB10+Xjn/e/eSVb7X408BMDZTVu75R8iaH/Wy7rt8t54A3OlkpzP2i5HT0+erJmGvHSRE96WbR76Tiy7qIZnsxa4mYInriTyot57dsVj3TflM4aOSlXEEcg9Z777n8fwSfRY7dHG9jV24a/1yEeEZeYn6Qu2zZGnHuprFvI70UGczm5EoLrfOpZamB81sEnse1zrPIC3BM9B8wqmltSHytzJkYVfVq0B2HkY8gbyRPY83RpuoJXoeR2ibaEpacFtEm94MIkSIEGFLwhB5Bq0SC+es4KrTH+azJdL42f2nSiX9/HfSFVq+/yEAnPf4SwAcOvtdAL7/1SkAnLxXz2CsT869AYDFTob5ugGy2lNXx6SoAZZ++wkAQ27UGLz8BwD6l4t49pbr3NSuwT2zy2QhHzdc517/TtXAQ1y8vm8/yU1/Pl5exiKnKLpnf3kMHctFRi+tEFHu9YYAOvSRXlB6uX7Odc1sfNWzj9uvd95GVhc905PA7TMSPYOKlTFyOqeHit58gVq1Sfy6+DaXGSHC1msRxQjkeGVUN/e0RLXUMEeQlDCup1oa+5/pn9/cnsdhtEQP5K3R83hrLVLbcpvLhrCtOoFtejOIECFChC0K8wP3DIwxJyIhuc44Twmw1tr2m3FujaIwL4Nf7tmPxYeLdL/nEXkA516h/gVnlyqz5i+DlJJZ+4qs+BsOvAqA2797Khjr8qEqItu3ULH3BX+8Fog1g++60+kALP5yLADle50KwHa5Ehpc7YrPcpw20KtTYxIUpc6q3b274vg3vKT5HNvNtZEcrIylcc8rsaDYxeIP6Kz4ftZ6pYN6xdSyVbHCsPx+ykzKWqQx5qyQZ9AhK7EYbX2xLPLsvMQ0z1zXfMDrH1WuiXEGHYeLF/GcQXlNYlzfewadgmyixP4FXo6ioWwiX2TmUU9+oi6JZ+Ca2/jrUxpILQ03t9kYjiCMxjiCbdVajJAIQ5NqCNokmuoZ3AkcZ62d2uiVWxB1fQZQ8feXeGuICOFrPvkrANflSFTukktUwfvCwZKlrnhblcA+jfKZ0lgtgCdzj7haG8tLt6k62P/hj31UOkfPzFc67xOTFOoZsaMW4ynPfAZA573UDGfMhJjsyK5upchbJNG7pfOU5tljd80zp4fSPStKRHT7xXdgR5GsaWY4EJOdrli7Ihi7w35KV80qla7RgmJtBtk1ZQmflReby85N3AwyarVh5rr+0JVrYyGo1DyFmnxns3DPY9/ZrEdIsjrNpZZaV3eQ1i4xJRXApoWb2einX+wDyeoknc1ixw3UGWxiZ7MNVSCHQzrJeh433gyn/rmt0dks/MhoU2sc2+pn1NTNYFlr2wgiRIgQYWtgG90LmrwZTDTGvAC8CsTrY7yyWWbVRMyat5Rjz76D4rE3AXDtEapje2uOLPBqFyr55F4Vob36Z3kGn12+PwAHj/pvMNaYk6Q51OGiPwLw3TWvJzzrhkOkPfTxlypOe/Id6Qf98udql/n41foohl0gktiTwQAnuMY46z6S6ODahbKUe5yk6uYuLjRVWxWzygEKcW0uC/vqdWc9V8VVCWf30Wvt5sqLWOt6HqeUynvwlnSp6z3cIT8xpTNlvatMdi0uvQcBkJIrz8CnoVaE2lyWVnoCWce+zaUfq841xTGZHYIxk7W5rA7SQJvW5rIhArml2lw2JQwQtbn8YaIFtYlaHZq6GbQH1gNHxJ2zwFbdDCJEiBBhS2Mb3QuathlYa89u/Kotj/TsXLrusC+HfSnL86pOImRzf6+Cr2My9XPM6WoT+Y8panOZ+rcHAFhwyCXBWIPHSqLi1o8lHLidi60XO4mFwi8kZfHzI6Uie/vtzwKQ98gvAJh9kV7/2e4ikP89+qVg7OFFsvznjf0SgPUr+wOQM+IEXVCaKFboUzVTl04HYHnXXRNe9y0uAdJ7i1x2RjxrHIFsSkRgp7jYvOcbunXQXMJtLjNd74LKtbFOoyl5iZ5BedDMRjf7Npeeg6nHGSTrXQD12lzW71+QjCNI3s8gWZvLpqaKhqUkGtIm2pKI2ly2TmyrHlpTs4l6AvcjdTyAj4DLrLULN9fEmoJh3bKZcMOO5Bx9GwBHTR0HwKUFIpQX7Ky+xIXjXeOaW9VM5sT7JS7Xvud2wVi3fqXwyOgXFGJ67wyFlhY6IvirO5QtdNaryiq6ftk8AGalx6vQwqH9tIBWuqYyAH0OUM/lOe/pntouakBT21/zNF8rfJXuuqx1dLn/VdPUF2NhOxHIvhYgPpxUV9QXgIKOWvwXz9LPmuXKCvKVySXVWjy7uTDRWr+SlCozyW8GZctixHNKnuoffHVwWSBMp9fXheoM6kIVxxvaDOoRyCHJ6spgsQ8fh2sI4sZM0tksjGSdzYL33YQFNCxP05y6Amg71mW0mSTCmG232rypm9zjSF61u/v3ujsXIUKECD8oGNO0f20NTeUMOllr4xf/0caYyzfHhJqD5d/N5P5BR/O7V0T27n6HLOmHnCX+9eFqqXzYzfIYXrpaIZ5dfqQeyWdec1Ew1j8ek9ewcpY8g36vy9vo8ZFokXvOlr7QbpOkkOqt+CcmyjnaLleWdd586SGlxil89j5ydwD+9fI0AEw3Wbuzy7QXd/7qCwDaFSrVtLtvZjNJ10/rLWs9Pz2xdgCgpoNSS/t30jX/KxFxXLsssZmNr3Xo7sJEFb5qeLWuzyrQdatmxjwashJ1jEpDktXlzjPIcmEir4OUmi+rP2hxmZ2oaQTJCeRwz+MgLBSSow63uIw/19w2l/UJ5HrTbTRkE7yvqM3lNg3Dthsmaur7WmmMOcMYk+r+nQGsbPSuCBEiRNjGYIxp0r+2hqZ6BucgzuAelEX0KbDVSeVUo2Kxn35wJwC3jpUl2GecUjg/yJA1mbffZQB07vs+EOMK7v/RgGCs0bffB0B2gVJDn1kixc6TjpMDtKD8YQCm/03cQbcdzwPgrY/mAfDQLmpEU/yGeir4NpgA2fscBMCyysfdM3Ttp/OV1rnPRKWptu+mpje9XMvM4snzAfh2V5G8vTPqewYr1ss6H9RV1c2eqyhbLI2hdFeAV+Gs5M45ssiXeGt+taqhPWdQ4ax9ABu0udTxuspEjiCsUuo5g/QceR9Bi8smEMj1m9kk0SLyze5r6xPIyZrZpCVJJd1QMxsdJxbZxaMxTmBrNLPZVi3WjcFmW4vNtuvBNTWb6Hvg+M08l2ajcMehnPHh+1yWp3z9Y/6hjJ69r1Bo57lZowDY/ujrAHj8t1rYLx/zBgCLfndBMFaX4cqa7TlUhPCdz3wNwJLjVH8wwC2in4+VIN0hf1dG0D9Hi6Te/uf7ATD5SXU1637A0cHYa4o0hg+35HUbCMC7U0T29v9aP4tOlrpHx0ESqFs5Tc7Xd4u0GeydndjIBmDxOi3AvfN1rsY1sylbqk0hM1fksxeb6+KypHxns5o17jrXyKY0rsrY9zz28GEivxn4zmZpSZrZBCGSBsJEdSE5iqqQ1EUyOYpwCCihAjlJM5twGKixZjYNZQ4l7YEchJ42OGT9ZzQhbPRDambTVmBovMtdW8UGNwNjzNXW2juNMfcjjyAB1tpLG7gtQoQIEbZZbKubcmOegS+jnbi5J7Ix+Hb+GgZd/BrjL90LgN47KeSR/6gI2adcWui4tw8C4KY/yXT8ax9Zw3eO+iIY64p3JHu9l2tmc+ip6j432lnUz7iexo+9OROAyw+QZ/DEn+YBkPujXwPwjatE3u+KmDz2ZwtDzWx6K0w0Y6Ys//mL9Pp2/ZXK2Wm1Uk9nfy3vZMUy1RV06KZQUHp2bjD2nFUiigcX6ZyvQVi/xFn8oVqBQtfz2AvT+XaYWYXySkrjLPRwM5vSJjazSc9RWChZIxuI1RXECONQHUEjzWzCjWyg5ZrZNFRNvKnNbLbR9aNF0JYqelWB3EJjGXMUcC/qgfyItfaO0OuZwJPAboijPdVaO8+9dh1wLlALXGqtHbup89ngZmCt9ZoM6621L8W/Zoz56aY+PEKECBHaGlpiLzDGpAIPAocDC4EvjDGvhRrbnwusttYONMachpSjTzXGbA+cBgxDqf7vGWO2s9ZufAMOmk4gXwe81IRzWxR1VZWsWzyLty+4HYDyg48E4MbXFcff5YKPAfj+Fz8G4JdHypr/z3FKKV1QXh2MdW9/V3m7POYtACz48n0AdrjrQgBS37oSgCFlLk3UWa7TU5TiOcPJZp+6S49gjGf/p/TTXRzvMHSwmsZ8OFZNeeav1zz2GyQZ6qLqge685rJmuXgAzyVklBcEY09fLk9gv1Azm3VLdD57JxHD4WY23jPwzWzynLfiuQWAqpTE9M8S93mFOYOMHKdFVO2LznzjmoYb2UD95jWVSVVJG9YiCjeygbbRzGZDz2juIhM1s9kaMC3lyewBzLLWzgEwxjwPnADEbwYnADe5318GHjCKUZ0APG+trQTmGmNmufE+25QJNcYZHA0cA/QwxtwX91J7oKbhuyJEiBBhG0XzCsqKjDHxIfZR1tpR7vcewIK41xYCe4buD66x1tYYY0qAQnd+QujeHmwiGvMMFiO+4Hjgy7jz64ArNvXhm4oh/brw8lO/YadjVET2B5dKeNGCZwDI/uAFAC7ufjgA963S53dpR3EMP3GWNsC3Fyn9tMopbQ7YX2POHD8GgIWDlG20awfFvxc+8hAA7XvuAMCjnysN1Pci2KNz7KO94lv1Pjh1gCz6tO1lhb85ejYQ0z86vqcK2bLZ1Z1/GoCyYt3fcTuliWbN7RSMPW2JLPuOoWY2XlYip32iVZ6fmegZlK9UplLRztslzB+gvDoxtbLEeTBdg2Y2iZxB0OYy4AycQml6fc8grEVUkcwzSA3JUSRpaRl/rtkcQRI5ingrPpyO2ty4cWtJR2wl02izMNZimu5BFltrR2zO+bQkGuMMJgGTjDHPWGtbnSdQPXMGiw8/jJ1PVrXwrx84Coh1Mvv6r9oELuwiIvQXY7RgH+skow8a9ZtgrD8cKcLYp4399s6dAbhq0WIAbv63iOOrf6QQzjdPaMPvc4G0it795HsAjnEksfnq7WDsJTO0YPc9VAtuV0dSV7hqYU/uDijQomlSJKzn0zzLV2szKBgqEjunpDAYe7ETpstYn1gD6DubtQttBmkVazSGW8DLXc9kL0pXEZdbH+5stsaFifqlJhLIQZioxKWWZic2wAtXGwP4fSZoZuMX9yTNbMJhonAjG6jfzKaxxT9ZyGZjUgfDpHNj1uOWCvE0tgm1lk2qLcHY5PUnzcAiIF7YrKc719A1C40xaUA+IpKbcm+zscFgqjHmRffrV8aYb+L+fWuM+WZTHx4hQoQIbQtWqWxN+bdhfAEMMsb0M8ZkIEL4tdA1rwFnut9PBsZZVVK+BpxmjMk0xvQDBgGfb+o7ayxMdJn7eeymPigexpgOwCPAcFS/cA4wHXgB6AvMA06x1q5OMgQAJeU1vDW1mI+PlhV8xmRtlic4Zc57H1eP47c/UeXvmeeqOviJpxXhmtDryGCsqrobgBiBekpnpXv+83AVtP3nbe19o351lsZ4Xt7HSQf0A+Cuu7RvDneW+KIxbwZjr12ksFDXsw/Q+8+WkxVYrs46y16lgrbyjv0T3mfVenkWGf1V2JY3I1Z0tnal3nvaWnkPXrJ6tUsD7dYxJFntmtl4LaJy18wmJb/QfQ5xYaJAsto9K5CsDqWWesnq1c6ad0VmQZioAQI56GEcEMg+dbRpbS69F5AeV+2VrMisqamkGyJ3m9vMZmNQj4Ru8SdEaBHYhtVwmzeErTHGXAyMRamlj1lrJxtjbgYmWmtfAx4FnnIE8Sq0YeCuexGRzTXARZuaSQSNh4mWuF+LgXJrbZ0xZjtgCPB28jsbxb3/396Zh0dVnX/8c7KHBEJCgLAHEBBQAUGqQhX3tWr9ad1rxaW21Yp1Q6lbWwStirbiglp3RcUNd3EBXEFwAZRFhMi+Q4CQhSTn98d7zp2Zm5nMhIRkBs73efKQe+fec08uyTnv9v2+wHta6zPMrtgMuAn4SGs9Vik1EhgJ3FCPZzg4ODg0LLSOxeqPcSj9DvCO79wtQd+XAWFL+LXWo4HRDTIRg1hLS6cDv1ZK5QIfIC7OWcB5dX2gUioHOAz4A4DWugKoUEqdCgwzlz0FTCXKZtChZ3tGP3Qbtw4VS/+1PqJK+twcSfpe9zdRIh0+I5QU9UFnUdYY9Uig7eWEX4tXseZHaUq/+FaRsBg9WrSI+o1/DIBNfe4DoHin5Bv+fIAQxG5ZLcng3oMlybv47UCFWHmGxNSTB0gOQ/0cqmxq1UirFsj5ZX1lLp4GkCGSJXUSWYucVgFl0SXzTP+CtZKzSDb9CzZVyC9sl1ZipZcYq1Ntk5/Pega2zWVyGM+gxIxh57Flh6+ZTYWU0aZmieVvy1r9JLPg3gWRVEn9x/5S00rrpaQYz8D7POg5kRLIEfoXWCmJGhZ5HeQovHsasH/BrjazaQhm7J7Krm1INFDOIO4Q62agtNY7lFIXAw8aiYrvdvGZXYH1wBNKqX5IldJVQNsgT2QN0DbsRJS6DLgMIL1lW46e3YlRHSVhOfmo0wEY8mgRAM/887cAnHyBsIuPukiSvVfeLZLWa+dO98b91cR/A1D8gTCIbU/jv/5FGuHYhfvxWZKnsVpFnbbIom/DM/ucJtVhj/5tUmDOvWRB+wWpJsr/Wp7rl6ze8o3IZ89tLZVLlrFsF4fKvEIAerYP5PJ//PR7+WyVLOqpGbL4285mnVuJ4F6RbYyzUV5xptkMin+REBRZMrfgzaC4XBZ/u5ju8ElWW15BqgnLVUfQItIpMTCQbX9l8x4rvMXfPMsvVBeGbdxQktWxyFVHuiZWyWqXuE1UaKiOu1qaBkGsYUmllDoE8QRsMLymhGZsSAEOBB7SWg8ASpCQkAeTJAkbmNNaT9BaD9JaD7I9BRwcHBwaBZqGSiDHHWL1DEYgjOPXTPKiG/DJLj5zBbBCa21jNJOQzWCtUqqd1nq1UqodsC7aQCWbNjLj+WfoPVOUQmeniqXa/HBhCe/XTNRJs9pI2GXSOb3l84ceBQJy1QBPbpbvz7zwVgAW/VkUUOffJVy7DgdeCsBLUxYD8NBAuX7tS08B0LKzGfuYXwNQtON5b2wrWT11qYR3Dp72vblnOADdfxTLec0sGfub3lL+2Tk99L9nbansj/u2a+6ds+WpJcvCS1YXmKY7a61k9UZJNGea8FFpibwzbRrZBNEMKPa1tSwvlWNPi8i036whWZ3RLGTeBCmU1gwLybxqJJCtNlEEyWq/XDU0rmS1X6XUSVY3LRovuqWhlt+TREasEtbTgGlKqWylVLahUO+SYqnWeo1SarlSqpfWeiFwFJIV/xEpoxpr/n1jV8Z3cHBw2J3Yq3MGSqn9EfW8PDlU64Hfa61/2MXnXgk8ZyqJliCNcpKAl0xe4hfgd9EGadepgD/fPZIDL7gfgKfnCuP40L/9F4D/3iiW93jjOSy4WHIGXQ69TP7tHWDy/nOCJG+/OU6IYf1MHPzjtyUxfNaLkrx94B5hN/f/s/QrmDH2dRnrt6J/tKZlLyDUws7p3AeAV7+RfEOnryVuXzBc4vRt9m8DwLq54gzNNh7EUKMwmpIhiqRFWyQv0D0vEJO3ZafblkkiOaOFlMLaEtmOLeTnWGxM14r18ozMVhJiKzbsr+qMgLdhsdXXzGanPbYkM5Mw9jOO/TmDqqSanoHtXxCxzWWU/gX+3gXy/IbpX1Bbsth6P9Hgt1Rd/4IAEkmlNCz25s0AeAT4m9b6EwCl1DDgUeDQXXmo1vo7IBxN+6i6jNNq6xp+/8mdjG8uQ72/SrgBH/9GQiNjRskCeOJcqQS6+VURl3tqsdT775MbSGx2HHYlAK+XiDzDtX+URPDYu6YBcMdQYf/edZMwkpNPGgfAjOHSG/mMw4Vv8P7PwgQuyAi82o49JaS0dKFU8vxkpCJ+1Vs2gbalhQDM/kykSjaukZ+jlZGvSMuQBPmCDXLf0M4BoTrbd3jbSsMfaC2LfECyWuZh5Sd21JCstptBzfzLFsMrsJLVFWYzSDWbQU3JapmfygjdDHZW10z/lFeFhoVKd/p5BqGS1dU2wWw2tXKTxAuXQE5LSTbz2TXJ6nCLVSTJitruqQ0JvhzuvdAa6iFoGM+IdTPIshsBgNZ6qlKqZvsqBwcHhz0ce3WYCFiilLoZeMYcn4+Ed5oUq9dsY/Toj1hU8gAAJQOlzPPBgcLgHvG4/HvfxZLkzU+TH/eAz8cDgcYuADkmAbzux88BaH+/8DkK/jMMgMyPpAdyRo6Ell77WWr/i0zt/U37i/V/0bPfAnBVy4DXUdlPuAiPPCwJbSudfWRPGatVsuggrSqT/XbzGkkgt9lfEs8Z6+S6uSvEazk9KLxlQzVbl0u4KLuHPNeGqVoaYTpbpmp/5vwDRGPJegY7Kmta75u2y9g5HuNYrvV6Hu/09Tyuljn4S0krgmJm1vKPKEzn4xlYT8AfJrI8g5SUYAZy3YTpvLBWDI1rdrUUtD6y2H40RXQllp977/JyGo50Fm+ItSBhONAaeBV4Bcg35xwcHBz2LuyNpaVKqQzgcmAfYC5wjdZ6Z233NCYK2mRx3dkHM7nLQAB2vied3zbcKkoZb+z7BwDKqkWb6M+3iqrpM3+Rss/gROFZj4m20HOPiMU/4Sf5MY/qL9b5rDuFRNZ+8PUAPPDeIgAONKZTx03zACj6URK0+xzT1Rs73+QG7llTBASStgNNG8u0jEMA2FRxNwAl60VdNe84GSNLSwvN+SvFM8jRO2q8i5J1ci67ZahVnoWx7k0OY8c6GSO1Vb55N2Jx79hZ85fXMo4LjHUeSCALMay63HgG2T6SmU+yOliu2i9JbY9LjRx2Umoo6czvCViVUr9CaTDqrk1EyOcW1bVY9dG0ifzTagyrPpwV7z/lyG71RAPKUcQbooWJngJ2Ap8CJwC9Ec6Bg4ODw14Hxd6bM+ijtd4fQCn1OA0gk9qQKM7rwLtnj6H4UZFvuPMayR2se0V0gzrd9gIA8++Sss+t594GwLxR0pAmuPzz/mPE+p5bdIQcPytqG3NGnStjnynVQyeO7A7As09+AMB5zcUKXjNR8hIbFkmlT+FfDvfGVgViOVuSlkV+mRDAdhZIvsJWAJUXS9VR9n4DAGixWkhcm9aK15K8JSBdbmPsm7aKTlD71qFWevJ2IaVZ+QnrQSTnirdiS1BLfAqlAJtKZEy/Sml6C+MZrBbPQWWKrlFApTRyziApxpxBJJXSAKGspmdQ1+qhyBpA4bSJGk57KBIciSwRoCHG8uJEQ7TNwAsJGcnV3TydumHFsjVcc8Ud7JgvIZwHrhU+wbWlQwAoWSclpd+dIPLU142Tnsj/+ZV0iFv1cyCBvPwm6XH8wChhJx90imjkbR9zLwBrykS76PohUmI6/h/zARh4sDB+F7wkjeBKk/cDIHXopd7YSYslKW31jfJMH+Jqk6xe0kc2K1vPX2HKW1O6HyDXz5GEsg1BVa1a7I1tx1xv+hF3ay2hpxIzVtI2wyvIl+usZHVyriSh7Qa0vTxUlA4CYSJblrqzzOgf+YXpMsKHifwcguBz5VXhE8bRhOmSmsWwGdRTmC6W5jb1FaYL7d3shOkSBlaOYg9EtM2gn1LKKJmhgExzrBAJoRaRb3VwcHDY87BXhom01rsqRtcoaNYyl/1+exb9/itkrQ//83sABp4iSd7zrvkjAL//u3gOGxeLKughbwpRrHzay95Ytvz0urMk+WyloO/9tAiA3iYc1GWdeABWpbT3uUJgG/8nYSarfeWV/UygNWXVUIGLAAAgAElEQVT+xxK+yi4oBKDQMIs3fvElALNbHgkEPAZrKVa07iHP6CDJ6h+myfwrlwUSyGlZsh9bldLCfLHSl9jQzlp5N83yJdS0ZYl4GWSHSlZvLgtVKAXYbjwDq1JqCW41VEqzQtnLsZSWeqWju6hS6lcoBadS6tAY2HsTyA4ODg4OwXCbQfyhV4tqPj1uBzl//RCAymVCJus0WLSHHh0kCdBm94hKaG6hxPNHfCnnrzwqoLW3qkzyCzOuF12jnkf+HYCX3xYJi9d/IyStokceBiC/p5SpZhwnieKiHU8CkN22EIBX5q3xxj5+irTMbNVNvIhui8WSXvHZQgA+7SkJ40Gm/NNazyu2imW+f0eRinjWKJQWLwnkOjJyREupxMTgOxur3aqUVq5bIe/AqJRuNdZ+daaMaY32TYYIF5wzqDCqpanZoVpEaS1MW0sbg/fnDFJCcwYhpaVGpdSfQPbkKGJUKbUKpem1kM4iqZQmecc1k+YQSY6i9uPGgEswB9BkqQ4nRxGfWLlwOX8fMoKJ30kDmof6HgzAtyWyUL/aVTSLjrxdksInGibwzbc+CcDshYd5Y/2jl4RN3potInL//N+BAJz5h38AsO8EaQf9+rHSVa3/P+V4ToXRDzILTJteUqn01ozl3th9vhY9o24nSdK2w1aZx0/vCYn7hyUiTHemqTpKM30a5hstoj4mKVxRIumbrUttDyDIyJWfodSs6p1zJLw13yySpatFwC6zjczTE6bLDNUi2mw2g4zgzcAnTFdpqqHSmkvISRt9IJUVmjqqNMuWXeh37Az88QSSyv7qIbNwm3lXGUa05RVUmAS5ZSTXlkD2C9NF0yLyw59wjgV1FaYLN4U9NZmb8MJ0IdDoyt1PtVJK5RGlJ7xSqj/wENACqAJGa61fNJ89CRwOFJvL/2A04SLCGRsODg4OsUIjnkEsX/XDSKQnfA/gI3wNwAx2IOrRfYHjgfuUUi2DPr9Oa93ffEXtTJnQnkGL1GSO6JhDwc2iQXTJCcIB+OogsZY/WS+W9eTfGJXPdRKWuc7U8c955y1vrMMfk1LSNw+Tf09MF8veWpuL2orX8flGSd5ef7SEZx6b8QsAA1pIaGTDALH6p78/xxt7brGEpU4dICWtHdJFi+iNl6Q8dd1y2bytlHV6qbCDZy+XZO/wQcKBqCwTnsGWpQEDIbufeBM2EZyXKZa2TUaXrJSftXlnYVJbLaKKlEyCsXmHhIAyg+IfFaaZTbr52bwEsqdFJPNTaaFj2YSxP1kcfM5jHKeEZxzbsFAgcevTJvI8g5r2TCSeQV14BdGuiXRPpGfsim3stIjiDxpdJ4+xHojaE15rvSjo+1VKqXWIbNCWXXmg8wwcHBwcYoVGOp3F8gX5SqlZQV+X1eFJMfWEt1BKDQbSgJ+DTo9WSs1RSo1TSqVHuNVDQnsGGb33pfdH0xmTL4nh69fPBeCJNhK3v/S30mjmw8OERayNtTn4SmmG8/WkV72xprcR5vEJnST+/cOoWwBo01ca4tz0liiiFhir9+Bmsvle/oXoCF18iFj9BYOkxeYbEwJtL9ea2Ps5XfMAyGp+pDkvIrDFq5bKvYdIkjprQWcAvvlFPICbhnYI+bm3rtjqfZ9zVGiLyVwxtAOewRrpr5A/qC8Q8Ay2V4RWRKwzDOZuyZFzBjaBnNrC5gxk7Gq/Z2Ct+ORQtjEEtIesZ+DXKvLnBGzOwJ8jCMcutufSvHvCl5r6y0T9CeddKQNtCi0iP/Zmi73xUKcE8gatdbi+LQAopT4ECsJ8NCrkiVprpVTYnvBmnHaIovSFWnulTjcim0gaMAHxKv5R22QTejP4celaDrzgfn5+RBb7vleK/MTn10mFT/O/PwTAwy36hNz31p+kcc0xJgwCMOIRack8Y8yZQIB3cMz/ZKN597WvALjB8A22vCDcgVXzZPPofZFsJr0KJTG7s8TmbQIVO4VKFvdKI6xnk76lGyXBnNNfwke5WyTst8qEj5I3y4bjSU9sDMhatM4P3QySt0oVU7M8WaC3rTYSFq0kfGWF6exmYNf+TaYX8gFB1TlWmC49xzCO1xouQpbMz5OfSA2dgw0TJdWSQPaHibzNwRcmSktJ8R2HLvRptVQTee/EL1EdpXooXILZf66+i31DuOSxbFqO39DA0A2XQNZaHx3pM6VUTD3hlVItgLeBUVrrr4LGtl5FuVLqCeDaaPNxYSIHBweHmKHR1VUxfdUTk5Fe8BChJ7xpG/wa8LTWepLvs3bmXwWcBsyL9sCE9gySklPIyGnNiGzpR7xluYRd5tx8JwA3j54KwEOHSdhl7UIJa6wacR4AT/7zSW+sfieKuF3JzfcBsLxUeAd/P1pCN8/++0EAhgyVZO73j4nOUUmahKTSjr4NAPWT6CMlB4VOPC2i7z8CYHHf/5N7/FpEfSRJ3WqBlJoumSdlodXLheuQkiklpmvKAr9ovduJZ2K1iKxnkNVWrPUS02IzJV+8UStMt9WUato5LNsuYaLsYM/AaBGlNTeM45VGi6hZaCmpX4uoRgK5qmYCub5aRH4dopBzEbSI/DwDi8bUIqqd5Vz7GE6LKA5gq4l2P8YSpie8UmoQcLnW+hJz7jCglVLqD+Y+W0L6nFKqNRI9/A5pRVArEnozcHBwcGhcaJsc3r1P0XojYXrCa61nAZeY758Fno1w/5F1fWZCbwb7FbZi+uMXkDfkLwDc+6Cok144QnIFJeulPPTAz0VvKGn2mwDcfNRNANx6zMPeWJm5kqy/5k0p9zzKxNw7zH8bCFjlfS8/FYA7z/oPAMkHyHXflQqruMObrwGQ03Ffb+yeiz8GYPWUqQB8ZhLI+T4totK8bgAM7Cr3zf3kawAqlkjcP81Y5JuDYvA92sq8Fltl0RVSTJDdVkpON/1kylCbS7mqTe5uMKWk1jPYZnIGmWkBOSpbSpqWJ2N5CdbmwaXMgZxBRM8gKIFstYgs49gjmVmGcRQtIss4jqRDFO5cXbWIavvcaRHt5dB1IyMmEhJ6M3BwcHBoXDg5irjElrnzmdxlIL+5U9panjNb4vq3pYu0RM+jTgfg8LtFruKKk4cCAWv4nSue8sY61EhWTHlNegzcefUwAOaMkfOdDxL5CY6VRjpryqTPgdU7evQrIZ+d/6boIHU49lRv7B5rxSpfNvUnAD7oJXH9/8sWK9lW1Py8WeL2B3YWy/vhzZIz2DxfiGOZuVKltj3I0u6eK1b5atuacnURAFkFMkZxmRxXZck7sZVNG32eQbmPYAaBZjzpLcX7qN4p9yQ1C1UprU71q5TafIDMqbZqIltKWuVTLd1pchpJttRUh88ZBGsT1VeLKOBtUANOi6hpETepjgasJoo3JPRmUF6lWVqyk6fSpffxzZdJQn3y4lkAdDPdvToPuxKAGxdK05tPrzgUgDvv+dQb65nzpKyz3SOSOG71pCz2T485CIDhN0h56gvzpMKrwIjKdR0gXIZPv5TyzwELJfk7dGSAG9AlSUJG7/xHnvfzYklkdzF6SBkZslnMWCGJ5KGdhTFtE8ubFknpaVbr0CQwQCdT9mmT1NuWyQaS3UF0kDaZRbe6WW7Iu1tvNoMMs2BXWKE6I0oHQVpEHq9AktFJPi0if1jILv5JvpBQ8DmrReQxis0YdnMoN7pHnlCd3Sxi4BlEKy2tUUoagYdQ27loWkT+EfzX78mJ3D1Li8gP5xk4ODg4ODReNVGjo8k2A6VUMjALWKm1Plkp1RWYCLQCZgMXaK0rahuj3X7dGfnm8/y1q4Rkjm4jic5Wd0jLyfXbpDSy8yGXALDsS9EiynxQEsf9HhvojVXxwHUAtOgomkN3fiUW9ioj+XzvYCkpPfIeKSm9o4ewidsOk6TvTbdIqGqRKdE858CAZ9AmX4oCfh4jpaXrl4qsdIchcm/WMil9/ewnkag+12gUWcbvpp/E22jZ1/RSDuIitmkm/4Wt08Uq3248g/aHiadjVUq3VYZaa2u2yLspMGEWGybKyA2EfGwCOb2lhIV0tTCfdWoo49hfJrpjp0+euiLIMzAM5B0RwkSep2COU2qQzEKT7rUlkL0wUQTGsTenehiyDVBP7qGuBnVD2N9Ry1kb4Bl7EjTaCzfuaWjKsORVwPyg4zuBcVrrfYDNwMVNMisHBweHSGg81dJGR5N4BkqpjsBJwGjgb4YldyRwrrnkKeA2RKs7In5YW8EB9y/jiWPFwu7/7NMA/LW1JIptovDdNccAcNpYiYef9l9pN/nGTcd4Y708+gMABo6Rlpj/e1EUXy/NNLo8r94FwOIZYtn2u0wkL/r0ltj8VaaM1UpMDAgO0edIKam10revLQKg7TnimeRWtQdgkWlak1m8IuTn3LhKSktbmzLSYKSXiDeRkycW/VaTd+jcWjwT2/Sm2PYDMO9k3TbxYPZJsQlk8YBCEsgbxHtIzhIvyFrB1emh8yizOYNkX87Ap0MEwf0LDKnMV1qali6/ktUmYRxJZ6hWbSJftjdSwthLOMcgR1EjR1BHLaLGkJ9w5ayNAK3RO2sNWCQsmipMdB9wPWDLUloBW7TWVixoBdAh3I1G+e8yAJXWnKVfTmHr0y8C8Ktxkjj+z8Fy66qfZXFV/xAnY+IoqQw66BRRgk35+F5v3Hk3CAdh/JkHAND38ScBOHqIhIdm3ikhpq3JUj3U4kzhNCQtFzkQyzi2iVxmBeSxF/c5Ta4xf6xlpmNZ2gARwWtbJKJ3RT9Kcrq6SOSvUzJk0V1pQjj7dRDdo7Kgv/rkzbIJZZkQmdUiSimQ0JNNNheX+RjHW2Whz0k1i3OphIQyWgZCQNVrTPVQ89DkszY/q59XYDkEdjPwwkbBYSJzTbkNE0VgHFdHSBinJfvDRkHaRA3EOI4lAVpfxnFMUtGOcRyHaBzSWVOg0cNESqmTgXVa69m7cr/WeoLWepDWehC+2LWDg4PDbocLEzUYhgCnKKVOBDKQlm33Ay2VUinGO+gIrIw2UPfCAsb9bxSnXnQHEFAK7TFVQj6HrhQG77UHXATALX3HAJBdUAjABc8Fmv8MN5Z1x6+F3Z3eXEIj/W+Ue28/4XYAUg4Ua/2L7eLUdJsoUtW5hcIB2G/JJwCseOMdb+wp6RK2ap8hISdrLRa3lGY8h/QQjsKcj8TLKFuwDYCMHCk53VAhnsH+xjOYH6wftEz6W7ToKPMp+kRKXMmRJLRlHK8rkbBQJMZxZal4FOmdAxyCKssriMI4Lq8MX1rqZxvLuYZlHIdNIEdhHEdqVOO34oOPG4Nx7HgFAcStQ6N1gxYNxBMa/fdPa32j1rqj1roQOBv4WGt9HvAJcIa5LKxKn4ODg0NTQ1dXx/SVaIgnnsENwESl1L+Ab4HHo92QumIpHW68gE4DRZuom0nmHnyNxOtPOF7IXgNMD4InrpdmNn+ZNBmA+/8d0Hia9ICoxX4xUkpEe58xGoD1/Q8BYFOFbXYjxLU7p4hFftVL38qzLz4fgF47JF+x+F2vIx2Tu4uTc3VLSfLaXMDcddJC8+BC8ULGmb4GG+ZIeWhWa2E7W8bxvvnivaxJDegHlf0iWkQtOosnsKF8CQBVzUVryZahrisJbWtZus0cm1JSP9sYgpK0Ps+gKkXusZ5AmZcMFs+n3DuumUCOlXFcXRU+Z2A9BT/bOPiaaIxj7/oIjONwbOP6Nrzfk2P3ezbJzAetvSZZexqadDPQWk9FenuitV4CDK7L/RuKy3ls8iLmrTCicOsl3NL8iekAPL1Awi73vyUNfq41/Y3v6yEhkbFG7gGg6NfXA/DOfOEg3H3+gQCM/lgW2wPNQr55qKjIfT5FuqrNWC619xcYvkGPNrJ5TLniBW/sZQtFTqKTkb9uVirVQ9OWCBP5QsNJsGGu9fOkL0VOPwkT2Qqlji1kcbWcAoDixbLR2B7HlnFcnhYqGbHaJIyzzEpXZvgTllew04SJMloG7vN6HDfLCRmrzMcr2G7CWN6xaYoT2AwCTYRs9VDlztDOZmWG4p/sLfa2mig8r6DW5jb1ZBzXtrg1VI/jplo/Ha+gftBaU72zMvqFCYh48gwcHBwc4hsa5xnEIzp0yeOOm89nXM/fAIFwwPUmDDT+gdcBGLujHwDnHVkIwNTT/gxAj2Nv8ca6aIK0vTzU1LcfukOSy+e/I97DiDNFm2jgUcJQHjJe+AirysRKuHxfseKzCqRxzfLSp72xNy2V/sldThsAQMvvZIyP54lg3ciDQks3LeO41fGh9fx5SZIEbt0soB9UXCTzaz1EEthbjdW+uSzUgl2xScJAfT3GsVjiXphouwkT5QW8AK/HcXpWyFillbXzCrabd+JnG0PAE6iOxDOI0NYy1v7GAKkRuAiRehyH4xX40RQ9jms8M8rnDo0Dtxk4ODg47OXQWlPt+hnEH5Yn5XJV5ukMS38FCJCzrt8k6qXdbpOk8F+vEyLzjS+KZPWVbQ4D4L5XfuWNdcrv/wnAv4zm0LfXS7nq2vU9AOjxguQUNELyslZnpnFH8opE+nplJ1FEDdYPsk12Wgy7AICCzZK8XWPIZkm/iBdiiWvLi8UD2N8klm1CNGVjkYzTMRDX31IkeYbUdoVAINm8xUcyW1kslv+hqaGegc0ZVG+R46ScVt7Yulokt6vT5Xn+Npa24f22CM3twyaQfZ5AqmUcV4Z6BlaXyU8y87e0TE2qmTPwk8xiaXgfjF2x+muUr9b4PPoYe3KSeU9CIlYKxQJX2uzg4OAQK0w1USxf9YFSKk8pNUUp9ZP5NzfCdVVKqe/M1+Sg812VUjOUUouVUi8qpdKiPTOhPYNNa9bx/L8f4OHlIkOhZkqO4JZjRSri1ufk57/KWI/D3xe5B9vS8rB1n3hj2fj30DvFm7BtLdUBUqk0O036FnR4Slpm2qY2/YpExXTVRKkeevc0ua59RuDVWiu3pKNUKB3aR8o/n/xKylLL5plKHkMys3mIA7tISadtaVm5dB4ALQsDcf1ln4mOkcqXSiUrP7HaaA9Zz2VjsZGfMPOylUsZ7WSsqp8MwSzIM7DQRosoUEpqCGI1WljK8TZTTZTsVQ4FkbeSff0JrBxFFJJZuOohCF82WkOlNEYSWm0tLaNpE8UDnGex+9GI1UQjgY+01mOVUiPN8Q1hrivVWvcPc94Kf05USj2MCH/WqvWW0JtBy7atOeaqy+nxx5cBGHacLNBHGV7Bf/4gWkSj3pEeyP+6XZK+E574EwDTLrnTG+uAC0SIbt0QCR3ZTmYF/Y4A4MbJPwBw7RNSrtrj8t8B0B/RAPrxJQn1TGwr5a3X5jfzxra8gq+N4NwRPWTRH2/CR2tnSilpdtvjgUB56G/aShOZjYZXULp4AQA5hW29sdd/sBSAqpx28q8JT6008t2ReAWVZWYDaiXPsF3M/JwCqMkrKPHCQJLI3lbhLyUNL08N0XkF/gSxn1fg/zwphGewa53M/Mfh1tSG5hUkskve0LyCRNvDqhsngXwqMMx8/xRSgh9uM6iBXRX+TOTfSQcHB4fGhSktjTFMlK+UmhX0dVkdntRWa73afL8GaBvhugwz9ldKqdPMuZiFP4OR0J5BodrK46nv0WmtWHwvjZMk7hPfSwL5lu6nAHBrlfRAvs2wbKf2PAuAtxYGVEufvkT4ble8ImSyC41WUfoJEvZ58dmPAZi+QkhmI46X8z27iQz25HeErLbUEMa6H9fNGzt7Y6E87wcpA71+mBDXbKhmzWwhjrU6LFRPqLClWNoFGWKRb14knkRe7y7e2OuNhb0jJbQMdcVm+VlbGEt6x3bjGeRLiMySzJq1kVBkdaUh4GXXDBOV+khmfk/AKyW1YaIyk4z2EcwAUoyXU1YSnmSWnORLIEeRsA5OIFv4LddUn6sQK8msNg0aRzLbS1E3BvIGrfWgSB8qpT4ECsJ8NCr0kVorpXSY6wC6aK1XKqW6AR8rpeYCxbFOMBgJvRk4ODg4NCY0DVdNpLU+OtJnSqm1Sql2WuvVSql2wLoIY6w0/y5RSk0FBgCvsAvCnwm9GaxYuoEbLvgfM9eINX/a2KkA/PppIXO9epvE4P93upSJDr1D5I7+dLdcd2lGgLzV/qP7AfjyTXklT4ySe4ceKRb+Q/8YBwTi+Sd3MXH0Tr8HYFXZAwBsXvI9AF1GHOmN3Wa6jPH59+I15A8ONJABWLdYSGbtzw+N17coExmLgtaSf9i0UH6u9scHxt5skrMbSkOTpb9sFN0jW0paZrSJmplcRpUhoaW2lGfqatFFqs4IlbEAKLFtLJOt/ETtJDObM7BeQHAC2TavsaWlVjU1Vk+gtsY1nreQHJpniNa8JpbkcFM0r3EksziE1lRXNEoCeTIi2DmWCMKdpsJoh9a6XCmVjyhC32U8CSv8OTHS/X4k9GaQ3yKdi4Z055cjZHH8ZqaEcpoPvQqApa/9G4BFN4uc9OQLpXFN80dlUzhr+ABvrLeuEinqrR0OBqDZpQ8CkDRNmMQZOSKC18l0Pqt8Wz6fNfhyALLNIla6WRbslEMu98butUE2gZmfSJfPyrkiM21lshf/JCGTISaxvMGscGqFMJdzu8qCvXmJ8BJSO/f0xra8AitEZ3kFizfJZmCb7ZSXSFgoq40s9lVrTBez3DZmJHmWDtoMvB7GXuWPWfy9aiEbFpLjlDTZ5CI1roGazWtqMIwjNK+JtXENRE4gRzym9s0hHOrbvMY1rklQaKhuHJ7BWOAlpdTFwC/A7wCUUoOAy7XWlwC9gUeUUtWIPTJWa/2jub/Owp8JvRk4ODg4NCY0jaNaqrXeCBwV5vws4BLz/RfA/hHur7PwZ0JvBpWdu7Hl/om810fKQXf2lSYyh1wpIZ/fjRLewacj5PyiSyRx3PmQSwDoNCYQbrl7vJTqthwi5al//2AxAGfcKzLXXQ8ZCcCvSz8F4Lvx7wPwyM7jADghR6xim0RdVBkI+ZzWX6zbD58VT23D59L2MrutaCatNZb2iV0kmftlmvy3VCwSHkJuD/FKFn0tHkZVbidvbJtsXmZ4BNZD2brFHFtVUpOszuwqz6gslzBRwDMQWLYxBHkGO0Mlqv2MY0+l1Fj1FebYholsGWnwORsm8kpHd4aGiSKVktpksI3bpoaTsI5QShoIPYUex4L6hmhc2V4ACe3Q6MDv1Z6GhN4MHBwcHBoXeo+Vo0jozeDnpas59aI72PyxJIivHXYjAJ/8VsosM58XgljpPeMBeKqTWP+PLxBtohHv/+KN5fUrOFW8iJcnCau55UxJrP75rr4A9O8lBQAPmn4FX88QBvANRhE1u1T+fWWeLREO9Cuw+YSVXwhRrFW/U4FA3L+3Se4uy5T/lg3fSYOcvH1lzDVl3wBQltW6xrsoMjmCFkb/f8dWYSBnmRLZCuMZ2FJS26sgybCeLXZUBirYrOVf7Csd3WpKR62W0nZznGJbaEboVQBB2kReziC0X0F6NJXSGMpC69qvwB7WykD2H0chmTWF9RuT/lGdx0xkM343wElYxydSs5rTaeAwjvxCWLQv3iKdwR4bKF3Hht4uDOSTbpGeyBeZheWgWXL+jOcDP/4/bjlBrj1VFv2u9wtvwEpD3NBHZBtUT5G/LhouieV186XPcs+/yrPbTt8HgHdmLPfGvmm/0CDBqrlSJdbht6FyI/k7RTK6XStZZNfPk42m7VGyeW0w4Zn1OwLVDDZZumR9CQC/SjO8AiNHkdVWNgNbPZTeRhb/6kqZQ3VmaOOaHUGcAFs9VGwTxukyr2LTGMdWD3kJZF/1kL9yCMII0UU4jlQ95JenTg36vNrjHtSveqgp5KnBVQ8lArTWVDVONVGjI6E3AwcHB4fGhQsTxSX6FmQw44Z9yfrN3QB8+r/bAFg+Rizp984S8basJ6Sv8cUjJTn/7OUiZV1ceKg3VtXw/wKQ+4GElJq1ktaU3bNMg5ZnxwLw+bCrAchJDS0lTT7ibwAcuL0IgI/f/sYbe+fshUCgPHXhIrGCj9tfyIerrPhakegb5fcSFvCGheIppHYVb8WGk6zENQS0hxauk9LRk401XrZVwkLZRohu5yrxHJJb9TZ3SgVadTPxTryWlUGeQbIvTGSPt+ywYSFJmpd6noFhHBsPpll2WsgxBPEKTMI4MzU0TBStlDSlFp6BRV2F6GLhGdRXotqVku4hcGEiBwcHBwfZDCIpQyQ2EnozWDtvMeN6/obb33wbgD9eLVb9mpeEdDZ12BkADDSKpJV/lBLUb26T8tEOB53ojXX+M1LGee04SQzvf7noFh2dPROAGXdLKek95XLPNfkSi/+vUST9bL38gpw7SMo+X334OW/sVVNMg/tOwmpe9alY0hcWigfwsbHmS76VhHfr/cSjmfuF5AwqWxUCgTLSnzfv8Ma2paTbTE4g27CVd+4wCeM+8gxriafkh0qhVKbJ/D1doaAy0OQ0SaoXl/sSxuXhcwTRFEkBmqWFTxjbeH+m73N/KWmyP9kbZC3vqippNEXScPfsqaqkuyNhvCc5NBrdWKqljY6E3gwcHBwcGhUadLXzDOIOqUrROj2ZYe/9C4B7c4TENUoLmWzHArHup/5VhAMPvVMa0YwbLPmAQ/4YaHtpW2O+Y1pR/vdckaroe5hUDz03VHICC7+Uvgb9hgu5L2+pPPORz6Rc9ImzRPLCkrwAfvlEmtm0P0NKTK2F3ztfLO+lWULmWjtL+hV0OuogAFaWyny3qNCG9D+t3e59X2Cs8+2WZNZeLH1bSprdQfIU1ZVGpyonlGRmdYZszmBTaaAM1CstNTkCW020ZYfxMnykMntseyekGSu/qjJQfWFLSSNWD/lJZkl+0lmoJ+FXJIUY5Cd83oUfu2LI1tX6bQhjeXeUkjrUDq2hqiL8702iI6E3g7wDenP2Z58yIksSrB+ulO5kg0+VHhBTDpbFd+YxEtqZV9EHgCGvPwLA0IoAF+CP20QsznYG67PsIwB+2UdKRm0HsU1GiK792L8AsEtybFsAAB7ISURBVM/r2wCYPVNCOmn7C7vYNrQB+PFHWZiH9JMGNJXmrzhj1RwA2pm+y2u/l2R09z/JRrTBlLCt2i6LsdUdmr96qzd2fyNvXbLV8Aw6Splt5WJJGKe26Q6ArhY9pKrM0ITx1gr5uVJs2Wh5UNmqObfRyF8HeAVWiyg0TJTRLC3k2C9CBzUTxv6Ecs3Fv/aFPVw/Y7+sdbSFumYCuabuUH0TxrEs3K6UNAGgtcsZODg4ODhAtdsMGgZKqU7A00jnHg1M0Frfr5TKA14ECoEi4Hda6821jTX3l03sc+lEpo+QEtEd10mXt44HXQzA4LHCTL4qR3oP55z2fwBc942YYGfef503Vo8jrgfgpCSx1r++7j4AHr5cGskca/omW+m/BRlCLrv4cLHSr3hTEsxr35Jy0JyOAf2oollvAXByX2lW9KUt/5wl3kfBQPFgZjwvz9YdxYMpNb90CzaIlW/LWdeuK/HGzjMEtfJi8Uia7yfP2DlXQkkpbTubKyU5XZ0lCWWvlNTXwnJzUJgoxXgCxeacDQOVm+PU9NAwUfPcUN2hZj6rHwJhoKoIYaJI2kPaRyizyeJgzyGS9tCulJLWFbsjYZyopaQJOu3YsAeXljZF0UMlcI3Wug9wMPAXpVQfAg2gewAfmWMHBweHuIFGuvLF8pVoaHTPwPT1XG2+36aUmo/056xzA+jqyp3s2LiS1y//JwCLDhNS2axtUsJ55ANiDd9iYvI9/yZaQP8aLWWfSZ8GJCMefEz6GAw+XvoQ3H7C7QB80kVyBLdfIHH8vFWSML532s8A3HOytL+82JDPfnpTehZ0OOl0b+ztk8SSOMgkd9dnixW+6lMpZy04REpdlz46G4CtGaF6QfNWiffR2qiZ2jJSCOQIyk3Oo3nntubdSD5E5bULGcvmCGxyeMOO0LLRDdsDhLaUTJmvTRhbeYloCePKChkz08w3OGfgLy2NlDC2SPXLUUTJIYQ7F017KJrukJyLf+2hXZmS0x6qI7R2CeTdAaVUIdKmbQYxNoA2TaUvA2jfsRNTn76a/U6QSp/pR3cF4JtDhwHwdbIsskdOfxmAozfJ4j9qs/T7TQv66xpcJFyFpX2lp3TxzlsAWL9ANpQud8i+1PsNWZinTpUKoayuInZnE8bzfpRF+chBHb2xy8xzslYIK7lTb1nsV3wl8ym8VCS1N1QIU7poS2ijmu+XS4XT+Rm2cigQJmppJKl3LpB5pXUQhrGuloR2VXOpJoqUMN5gFnp/shgCG8QWcy41w4aFJIHcrIUwkCMljG2yOCSB7AsdZfiE6gJhotCwUKSuZeF6IPvXt6g8gyjsYrkmzMmQMWtPGMeiO5SoYaG9CXoPJp01GTdGKZWN9OocobXeGvyZ1lojHlkNaK0naK0Haa0H5bXKD3eJg4ODw+6B2Qxi+Uo0NIlnoJRKRTaC57TWr5rTMTWADkb5woUUHT6MAw3DuMDwBsbkG4bxpVJSesIkcTiuHSe6QoMMu/jMdj95Y31yqfQ4vvPKQgCuKZAmL08Y63jaTuEmXHusMHjPeEmmvex5GbvVPhKaWjTzTQAu7N/BG/tj0ypz26fvAtDxUCn3/HiCeB2HdhFpbZsw/n6t7I22ZeWMNZIMbl0g3keZCUkBtBhkylW/Nz2N2xeaTz6X85kSIrNhoc2lRmfIsIutZ5BqPJtNJUHJXhMWitSsxs9Abp5hwkLG6vcSyEElmtEYxv6Esb9Rjd9zCLb6I8pcE8VqryO7WMZMTDiGcX2x5zKQG/13Wslf2uPAfK31vUEf2QbQEGMDZwcHB4dGhWEgx/JVHyil8pRSU5RSP5l/c8Ncc4RS6rugrzKl1GnmsyeVUkuDPusf7ZlN4RkMAS4A5iqlvjPnbiJCA+jasLWskvcXb2baERJT7z5iEgDTTKnpldcLYWzgKVI2us8SqVR980/iQWSfe7831mOdpJ/B9+9NBeDwsVKG2mGGJIxvf0OYx1Mukmb0lmG84FVR/9znGmEqVzwrvwT7Nw9Y2Ovyxbsoel8SxL3+cAoAP4+bDsDqqmYhP9fXRTLPAcbS3mJ6FeR2k1aaZaaMFCBnHyl9ra4UL0fnBXIVAJvLjIVteg+sLQnNEaw3TXBssnhjUAI51XoGxpuwxyXmnkwzv8oKc+xLGIcjnfkTxhnJdWMY+3ME/kY2UJNhXMPyr3FHdESzfnclRxANuyNh7FA/aBqNZ2CrK8cqpUaa45CCGq31J0B/kM0DWAx8EHTJdVrrSbE+sCmqiT4j8u9xjQbQtaFDr47c8fgYbj5c+AKbBku10E9/F45Ar/uuBCC/p0haH/GLLL4bRv4BgEfPvMMbq5MJ5WxfWwRAxW8fAODctsLcHf+A9FMubfkhAM3bSajnq/nTALj08G4AzLMLoAkXAXQ5TGr9l3woY/e9V+azqeJOAOauC+URfPWLbAanmL7K2zdIxCy3l4SEKqYHUiypnUVqQ1eLlEVVCwlj2bDQFrMZWM7AuhJZuG1YaN02eyxho21BYaJ003Gt3HQya9laZDGsJLU/LORPGHshoaCesf7FP8V3nF6jmU3o59HkqCE6j2BX5Kj919Q32ZvIyeIEnnr9oTXVjVNNVNfqyjOAd7XWO2q5plYkaujTwcHBodGhtXgGsXzVEzFVVwbhbOAF37nRSqk5SqlxSqn0aA9MaDmKhdtTOOLTfG4plPBJ2zFXAHDOVdKycvhHnwLw3IJ7ADj4YrH+LYfgmeLPvLGmXSbicP9ZJf9e/7Y0pLE8gjEjpR/xtw8Jj6DrSbcBsP5daaF5vWlIk2ys+RWT3/fG7nycjPnaJLHeD8mRElj7+/JVkZSjts+Q+W1cLQnjPMOPsA108o6VkFDVhwFNpaSCriHvpLhK/kutZ7DahH1SMsSqX10sgnapWdL0Zt1WOU43zy4rCTCQbViozLbM9HgGYvlnm3uqKuRz6ylU1RImSvdKSSUJl5ESnkcQjmEc7jjYSt1VHkE8cAjA8QgSBXXodJavlJoVdDxBaz3BHiilPgQKat7GqJDnaa2VUhF3F1Nwsz/wftDpG5FNJA2YgHgV/6htsgm9GTg4ODg0KnSdrP4NWutBkYfSR0f6TClVl+rK3wGvaa09Sy7IqyhXSj0BXBttsgm9GezYvIlZL79Ajy8kbn/wGxKDH50k8fHCZmK57vuKeAJvHH8jAMlKjtfOm+6N1eEzUTI9+W1hFr/5sngN/00V/SDbBvPzL8TbuPgB8RiKxohlm/6t5Ah6/Vqa2yx+N1C22vUaUdZYWy6ksu/XSo7ANqb58qcNAFxt2kQWr5Xj1vtJjqD8a8khZOwjiW9LKAOozJXneRLUJkeQahLCq21OwB4bqeu0ZuIpbDLJ4DSbHwjSJmph9Jg2mtLSbGv5l5tGOumhCeNsXw4hy6dQCkHNbCqtSmkoqayGaqnv2F826ieUhTtX35aVUH9SWSLnCByC0HikM1tdOZbo1ZXnIJ6Ah6CNRAGnAfOiPTChNwMHBweHxoSm0YTqwlZXKqUGAZdrrS8xx4VAJ2Ca7/7nlFKtETvlO+DyaA9M6M2gfccCrrr3BgadL4Sx4R89D8DL82cAMKRIrPnbT5LmN8/MHQjANFNa+rjJDwD8+c3FAPz7JLH4nxwjvRFm3SVx/u7HijzF8o+eBeCK/SWf815LqcJZ9oJUcHU/9RA5/16g7eVB+fsCgaY2HxtPoL2xpN9bJmWqrfsKo7pkvVQw5R8pyqiV08XjS+5sm9kHqse2Is+30hErtobmCJaZFplpzSX/sLrYxP9NvL/USE2kW2JckO5RhjlncwQtTb+CWHME/vwA1MwRWE+hrjmCWquJ/Md1zBE0RlXF7mhM4whljQCtqarY/ZuB1nojYaortdazgEuCjosQbTf/dUfW9ZkJvRnkFa/mzHf+yX25QwA4KFcWxs73SeOZ8WeNAaCFWXBsWChn+mMAXPrFMm8sWzp6b8krQKB0dMrHsuFe87CwmufdJQtc+uey8ex/vFy34FVJLBeOvBWA5aVPemN/uUIa4NjS0U9/FG2kkUZ+evPKVQC0PVBKUMumy2aRse/hQCAsZHsh2+QwwLoSwwEwYaBlZrFPMwniFZvNsQkLbTQJ5AzTXa3MMJBt2ejG1du8sVuaMNuuhoX8ISEIlI42VFgoXJiormGhWMpG95awUIJOu9GgNVTrxJOaiAUJvRk4ODg4NDaq3GYQf1i9dhtj75rG4lJJ/qZsFcbxlW2HATBxgSRsV/1vOABPfiFNY04ZL5pAU4d388Yas0JKR6f9fSYAA0ZJeaotHb25i9iPeUYyev6DLwLQ+/IzAHjhJUle982wzWQCmDxXwjyDssSif930WW43UKrKbFiozenSvrPyAwlNJRUeYEYQRdX1FWJ5W/YwwNItRpMoS+a1xLCVbVjoF9MYx7aktHLTGWYum0w/5WYmJFRRGggT5dg2lmVyjfUUKk2YyPMMjOXfzCOd7Qw9Dkog+0tH/WGjtJTwnkAkTyGcIVvf0tGwiqIxXFPXMaM9I/qYzoxvbGgCJeF7GhJ6M3BwcHBobDjPIA5R0DabG87/NZM6SuOZ+/8iMhR3D5SSzOeNhfpKjwsAmDhU4uoHnyalngvnBprbdPqVeA/vT/gYgPG/E6t8yighkW35n1j+/S4R3aPX75SS0z7PnQPA+nKRtnjbSw6nemO/OldIY+f3FGt90zIpO+0wTDyVsudNjqDfCeYO8QxKc0RnyCaHl9ky0GYtvLF/3mQs/xbSt2DJerHiM5tLDmBrsdENai5W/g6TMG5jPJwKU0raypS1VpZu98ZuZbwH6wnkGM/A5giae1pEppmNL2eQ4UsOh5yL0L+gRo4gSklnsDZRRNXSXcgR1BWNkSNwCeKmR7UOFILsaUjozWBLy/a8duptVDwk8tFzJkvoZv/pslBfbRLEV98qFUA/nyYLXVZrqc1/8ZUp3lj//FIqjOY9IQtal++kIc6Rp4ow3cx7ZZM4fuZEuW6UhG6mLJNqHZscfvmLXwAY2SYgPvfgYplH52E9ACiZLptQzsGSIK5+Wsba2V6S1DZBvKxYFlmbDLa9kDNyWntjLzAJ34wcETVctVHmk2Uaz2w3CWWbIN5idJDyzeeLSuT+vCw5tpVCUHPxb+FLIPu1iPxhoXSf7pCc80lY1xCeI+TzGgnjXWhEkxwlbBQtORz2miiraLyKzLnFv/5wYSIHBweHvRwa7cJE8YiVy9dw44ixlCyUstAPXxem7qBr3gFg4UViBt1t2lw+ebWcv+yF1wAofv8xb6yzMpcC0HWIhGa+vEEkRA57RsI/jz4vpb3ttJT02paUj3wq7S8vzJVQzsQfpEy027HdvbG3LpDkdMFFolZa+YE0nqGXcBJU0nsALC8VO9d6AnPXidWeniP8g3krRa00IzcgZ7LI9EfONnwHyxOwLSnXGQ5DYTcJUS0pMY1ymsv1O3fI523M9TuDwkS5Pl5BoLRUPJbmvuY1/rCQ9RSC4U8QGypC1DBR4PPQ8cJJWEdLIO+OlpROV2jvgEsgOzg4ODgAbjOISzRrmcf+/3c2+98jekLzLpWkaPYznwDwzEmSOzjvYYnzLzpbPIL/9JUY9+emZSTAV5fcBMDgcdIb4cZDRwCQ30p0puwvwF0fiZV/qvEEXp+1EoC+J4onsHnJ9wB0uf4Yb+zyUV8DkDxAzqkkKW1dUS2tNa0n8P0aE//PFXbzN8ukBDWrtZSrzlkuxy3yAvmIYpMjyM4xDe6Np9Cpiyi5/vKDENbatRR1U9uUx+8J5GWFegEAORmhnkB2Wijj2Fr+fk/AWvHWCwjOGfjbWtZscF83TyCc8RwtgVzfMtFY7okXT8A5Fw0LrV01UVyiV4tKph2xhdzrRFTugcclDDTChIG+P0WOHzpAFtmZw0QCevpv/wgEQkAA1/aXMFBGgYRy7H/4TW9KJ7ML82UBvma6yFb847ciMbFhgSzsXf8ujXXKbpAQUNLBV3ljq6RvAPgFSfKmGw7ATBP2ybQieEtEytou/rOXynGOefZmwwlo0SrAM7BhoI6dZENZNl8W/465svh/tW2TOZZ7Ksxm0LaFhIns4p9reAZ24QfISQ9d/Jv7mtXYzcGrHkoNTRinJdfcDFKSGnbx9y/8UH+2cCwMZD/igSPgFv7dD42rJnJwcHDY6+FyBnGKlQtXcPPh1zFpzpcAfD1AZKRvLhOPYOVlIkw36TDxBE6fKyWcVxYcAcD66t7eWJmmhvGvz4kVf0s3seIv+kjaND90ufAL1n8gnkD3By4GoPwS0TJi6NkAJKVISGhBWVZgbBP2+chY/tltCwGYMl8kynPaixU/e7HwDfLamn7EJmxky0JX/LQRgF49WnljL/1OEtjdWouo3RemP3IX401YT6BdjngClWWmtNTqDlWIVlGubVQTpCOU49MeyvExjv2cAb8nkBrGnK6rJ1Dj8yghIKi/blC8soVrJsbrPaTDLsCFiRwcHBz2ckjOoKlnsXuQ0JtBi/QUji7MJe+68wG44U3pFmclq4evEKt++iP7A/DBJ5KAPdSom456ZIY31iRDLhv/gTS8//WYcwHY8C+x9NveL2NXTh4NwKquQhhLzZLrPygSK755e0kkv/T9Km/snM7CNH7jW0k2t+piEsILxYpvY+L961ZIDmGf3kIqm/OVlLsO7i85hYVfzAWgR9s+3tgfbF1vzok3YRPEHVqEegJtDKnMsonzbdmosfrzMkMJZgDN0305AZ8nkO5LEKf5TOo0nxcANXMGKclRPIGongI1EM0TqMFIjoF0Fs0I91v+u8MTcIgPOM/AwcHBYS+HBhqltU0TIKE3g/Reveg6ZSrj2orlX3ZefwAONVr9J9wmVvukMyQ3cPhjEt//76NSOfSnf73pjdXn3QcAKD1BLP8NR0ipaeq4mwF4d5PE7XNMg5kJM0RSIr+nNMh5eLrE7gt6idX+/syA7lHHnkISW7pQcgJ+y/+44+WeN2dL5dKBx4uX8uWb0wAY0FnyFS+bfECvNtne2BXbhGjXyZSWVuwQ78LzDEwvgramdNRa/nlWasJWCqXXbF6f6fcEfGZ4hi9HkOaTn/Bb/QCpSf5jn6cQxROIJf7v9yaiWf6xGOANbfmHu915AvEPjXbVRPGI+UvXMviCcRQ9/nsAWv/7QQAmfCsaRX867X4A2k2VLmQVpgfyFwdI2WfzdhO8se76QRawgn6SXL769R8AKBwsDYPueFVaiHY/SETxXpsiJab7DpJy1R9nyeJ/1NGykL/72lfe2MP/MAyARx5+C4DLzxANos8mCfP41/tIOeuLGyW0NLCTcATKi2Xz6JUvG1G5Wfi7BfEMLE+gs0kQ20Y0rX28gZa+5G+2rwy0WWpNHSH/ZpAZoXTUwn8cLoEcbbGvERaKdhxDArlmQrn2hX1XFvpoyV230O8ZkGoitxk4ODg47N1wCeTGg1LqeOB+IBl4TGs9NtK1SSmpNGvVgcuTRW6661CxrA+fKCWc/U6Tcs+jRk8F4JBzzgTgj/dI+8sTzz3OG+vBx+Sa358/FIDHJkgZ6o3Xng7Av0ZLT+Nxoy8C4K/XPSTnh18p900Uots5N4gn8cL9872xT+z9OwDuWVMk8ysU0lmp0Uwa2F6Y0+WGINbbeAK2LLSwZShBrH3zQNtLe65VZqjln5sRygbOSQ+16rPTQj/P8sdvgEyfjlCGzwxP9zemieIpQM2exlE9haiS1uEIYrVfsytWvLP0HaDxPAOl1JnAbUBvYLDpfRzuurDrpVKqKzARaAXMBi7QWleEG8OiMXp/xwylVDIwHjgB6AOco5TqU/tdDg4ODo2HKh3bVz0xDzgdmB7pgijr5Z3AOK31PsBm4OJoD4w3z2AwsFhrvQRAKTUROBX4MdzF+xW24vPHzyPnkD8DUPyl5AwiHc/0HU+470FvrJx75PtbjpL8wz1/F8v+TweJSunItUUA/K6vJH8v3SwNa47fR8hp1qof0kn0hmyrSIABBWLpWyu+Vyux9K0V37VlWshxxxaS3LVWe7vs0OPWzWr+t+VlhiqEtswIPbYJYovstFA7oFk4z8B3rq6egN/qD3cuWk7AWfEO8YRqGkeOQms9H6ISJMOul0qp+cCRwLnmuqcQL+Oh2gZTOo6SIUqpM4DjtdaXmOMLgF9pra8IuuYy4DJzuB+yg8Y78oENTT2JGODm2bBIhHkmwhyhYebZRWvdOvplkaGUes/MJRZkAGVBxxO01hMiXRzheVOBa8OFiSKtl8jC/5XxClBKdQLe1VrvV9uz4s0ziArzMicAKKVmaa0HNfGUosLNs2Hh5tlwSIQ5QvzMU2t9fEONpZT6ECgI89EorfUbDfWcWBFvm8FKoFPQcUdzzsHBwWGPgtb66HoOEWm93Ai0VEqlaK0riXEdjasEMvA10EMp1VUplQacDUxu4jk5ODg4xCPCrpdaYv+fAGeY6y4EonoacbUZmF3sCuB9YD7wktb6h1puqVP8rQnh5tmwcPNsOCTCHCFx5tkgUEr9Vim1AjgEeFsp9b45314p9Q5EXS9vAP6mlFqMlJc+HvWZ8ZRAdnBwcHBoGsSVZ+Dg4ODg0DRwm4GDg4ODQ+JuBkqp45VSC5VSi5VSI5t6PiD1vEqpT5RSPyqlflBKXWXO5ymlpiilfjL/5jb1XEEYjEqpb5VSb5njrkqpGeadvmiSUk09x5ZKqUlKqQVKqflKqUPi8X0qpa42/+fzlFIvKKUy4uF9KqX+p5Rap5SaF3Qu7PtTgv+Y+c5RSh3YxPP8t/l/n6OUek0p1TLosxvNPBcqpY4LP6pDXZCQm0Ecy1ZUAtdorfsABwN/MfMaCXykte4BfGSO4wFXIYknizpT2BsB9wPvaa33Bfoh842r96mU6gD8FRhkiD3JSGVHPLzPJwF/bXyk93cC0MN8XUYUxmoD40lqznMKsJ/W+gBgEXAjgPmbOhvoa+550KwJDvVAQm4GBNGwjfiSla1oUmitV2utvzHfb0MWrg7I3J4ylz0FnNY0MwxAKdUROAl4zBwrhMI+yVzS5PNUSuUAh2EqIbTWFVrrLcTh+0Q4O5lKqRSgGbCaOHifWuvpwCbf6Ujv71TgaS34CqlVb9dU89Raf2AqZgC+Qurl7Twnaq3LtdZLgcXImuBQDyTqZtABWB50vMKcixsopQqBAcAMoK3WerX5aA3QtommFYz7gOsJNG5qBWwJ+uOLh3faFVgPPGHCWY8ppbKIs/eptV4J3A0sQzaBYkQpMt7ep0Wk9xfPf1fDgXfN9/E8z4RFom4GcQ2lVDbwCjBCa701+DNDCGnSel6l1MnAOq317KacRwxIAQ4EHtJaDwBK8IWE4uR95iLWalegPZBFzZBHXCIe3l80KKVGISHY55p6LnsyEnUziFvZCqVUKrIRPKe1ftWcXmvdbfPvuqaan8EQ4BSlVBESYjsSic23NGEOiI93ugJYobWeYY4nIZtDvL3Po4GlWuv1WuudwKvIO46392kR6f3F3d+VUuoPwMnAeTpAioq7ee4JSNTNIC5lK0zc/XFgvtb63qCPJiOUcIiRGr47obW+UWvdUWtdiLy7j7XW57ELFPbdCa31GmC5UqqXOXUUImceV+8TCQ8drJRqZn4H7Dzj6n0GIdL7mwz83lQVHQwUB4WTGh1KGrdcD5yitd4R9NFk4GylVLqSJi49gJlNMcc9ClrrhPwCTkQqDH5GVP7iYU5DEZd7DvCd+ToRicd/BPwEfAjkNfVcg+Y8DHjLfN8N+aNaDLwMpMfB/PoDs8w7fR3Ijcf3CdwOLEAk1Z8B0uPhfQIvIHmMnYindXGk94e0ix5v/qbmItVRTTnPxUhuwP4tPRx0/Sgzz4XACU39/78nfDk5CgcHBweHhA0TOTg4ODg0INxm4ODg4ODgNgMHBwcHB7cZODg4ODjgNgMHBwcHB9xm4BDHUEpVKaW+M2qg3yulrlFK7fLvrFLqpqDvC4MVMh0c9na4zcAhnlGqte6vte4LHIOoat5aj/Fuin6Jg8PeCbcZOCQEtNbrEFnlKwxDNtno3X9t9O7/CKCUGqaUmq6Uetto3T+slEpSSo1FVEW/U0pZjZtkpdSjxvP4QCmV2VQ/n4NDU8NtBg4JA631EqRXQBuEoVqstT4IOAi41EgTgMgZX4n0uugOnK61HknA0zjPXNcDGG88jy3A/zXeT+PgEF9wm4FDouJYREfnO0QmvBWyuAPM1NLrogqRORgaYYylWuvvzPezgcLdOF8Hh7hGSvRLHBziA0qpbkAVorKpgCu11u/7rhlGTUnmSJor5UHfVwEuTOSw18J5Bg4JAaVUa+Bh4AEtglrvA38ykuEopXqaxjcAg42ibRJwFvCZOb/TXu/g4BAK5xk4xDMyTRgoFWlu8gxgpcEfQ8I63xjZ6PUE2jd+DTwA7IPISL9mzk8A5iilvkFULx0cHAycaqnDHgUTJrpWa31yU8/FwSGR4MJEDg4ODg7OM3BwcHBwcJ6Bg4ODgwNuM3BwcHBwwG0GDg4ODg64zcDBwcHBAbcZODg4ODgA/w/c9IjQ6nPJawAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kRJOG3puSMbl",
        "colab_type": "text"
      },
      "source": [
        "## Pad Mask\n",
        "\n",
        "![](https://paul-hyun.github.io/assets/transformer/pad_mask.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sC6NSo3cJ2HE",
        "colab_type": "text"
      },
      "source": [
        "### Tutorial"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D9lJOPCBnSj4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d33abd1e-a3d4-4fd9-c0cd-540582866f3d"
      },
      "source": [
        "# 0인 부분 확인\n",
        "mask1 = tf.math.equal(enc_tokens, config.i_pad)\n",
        "mask1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6), dtype=bool, numpy=array([[False, False, False, False, False,  True]])>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_6A0dP-wnfW_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7ae2be94-0ad6-497a-904b-3df714a34950"
      },
      "source": [
        "# boolean에서 flot32로 변경\n",
        "mask2 = tf.cast(mask1, tf.float32)\n",
        "mask2"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6), dtype=float32, numpy=array([[0., 0., 0., 0., 0., 1.]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vks9dn_eno6M",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9ec8ff4c-14bc-4bf1-c0f5-a90032d3652b"
      },
      "source": [
        "# 차원 확대 (차원이 1인 경우에는 차원에 상관없이 연산이 가능)\n",
        "mask = tf.expand_dims(mask2, axis=1)\n",
        "mask"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 1, 6), dtype=float32, numpy=array([[[0., 0., 0., 0., 0., 1.]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TnFBTppYJ4QE",
        "colab_type": "text"
      },
      "source": [
        "### Function"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UW5K7beRnBI2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_pad_mask(tokens, i_pad=0):\n",
        "    \"\"\"\n",
        "    pad mask 계산하는 함수\n",
        "    :param tokens: tokens (bs, n_seq)\n",
        "    :param i_pad: id of pad\n",
        "    :return mask: pad mask (pad: 1, other: 0)\n",
        "    \"\"\"\n",
        "    mask = tf.cast(tf.math.equal(tokens, i_pad), tf.float32)\n",
        "    mask = tf.expand_dims(mask, axis=1)\n",
        "    return mask"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "253aegyMJ5s9",
        "colab_type": "text"
      },
      "source": [
        "### Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KA1M4BtSTf6n",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1f163ce6-d90c-4641-db0f-c6611c6fe325"
      },
      "source": [
        "enc_pad_mask = get_pad_mask(enc_tokens)\n",
        "# 함수로 구한 mask와 계산한 mask가 같으면 정상\n",
        "assert np.array_equal(mask, enc_pad_mask)\n",
        "enc_tokens, enc_pad_mask"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([[4, 7, 6, 5, 9, 0]]),\n",
              " <tf.Tensor: shape=(1, 1, 6), dtype=float32, numpy=array([[[0., 0., 0., 0., 0., 1.]]], dtype=float32)>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1FbtXvu_S6Hm",
        "colab_type": "text"
      },
      "source": [
        "## Ahead mask (Decoder)\n",
        "\n",
        "![](https://paul-hyun.github.io/assets/transformer/ahead_mask.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H1bAp9rqKAMW",
        "colab_type": "text"
      },
      "source": [
        "### Tutorial"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g88v778TzRZV",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "08dca9e8-541f-44b2-fbdb-eee1f60c42b4"
      },
      "source": [
        "# sequence 개수 추출 (bs, n_seq)\n",
        "n_seq = tf.shape(dec_tokens)[1]\n",
        "n_seq"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(), dtype=int32, numpy=5>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zztMXOygzjgF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "068e4810-4d3d-455f-f7c9-066a77e74b06"
      },
      "source": [
        "# 전체가 1일 matrix 생성\n",
        "ahead_mask1 = tf.ones((n_seq, n_seq))\n",
        "ahead_mask1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(5, 5), dtype=float32, numpy=\n",
              "array([[1., 1., 1., 1., 1.],\n",
              "       [1., 1., 1., 1., 1.],\n",
              "       [1., 1., 1., 1., 1.],\n",
              "       [1., 1., 1., 1., 1.],\n",
              "       [1., 1., 1., 1., 1.]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iGMP_M8RUQdb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5d0fbfcd-04f2-4c37-e2ea-6343ebbd2b23"
      },
      "source": [
        "# 중간을 기준으로 위쪽을 모두 0으로 변경\n",
        "# https://www.tensorflow.org/api_docs/python/tf/linalg/band_part#useful_special_cases_2\n",
        "ahead_mask2 = tf.linalg.band_part(ahead_mask1, -1, 0)\n",
        "ahead_mask2"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(5, 5), dtype=float32, numpy=\n",
              "array([[1., 0., 0., 0., 0.],\n",
              "       [1., 1., 0., 0., 0.],\n",
              "       [1., 1., 1., 0., 0.],\n",
              "       [1., 1., 1., 1., 0.],\n",
              "       [1., 1., 1., 1., 1.]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1mjydNumzpb_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c3e8199b-37fa-4c1f-a647-325c8e512a92"
      },
      "source": [
        "# 1에서 값을 빼서 0->1, 1->0으로 변경\n",
        "ahead_mask3 = 1 - ahead_mask2\n",
        "ahead_mask3"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(5, 5), dtype=float32, numpy=\n",
              "array([[0., 1., 1., 1., 1.],\n",
              "       [0., 0., 1., 1., 1.],\n",
              "       [0., 0., 0., 1., 1.],\n",
              "       [0., 0., 0., 0., 1.],\n",
              "       [0., 0., 0., 0., 0.]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nesuHASz0GqL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b90aa8c2-2218-4496-ed02-5c08d223055f"
      },
      "source": [
        "# batch 차원 추가 (0)\n",
        "ahead_mask4 = tf.expand_dims(ahead_mask3, axis=0)\n",
        "ahead_mask4"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 5), dtype=float32, numpy=\n",
              "array([[[0., 1., 1., 1., 1.],\n",
              "        [0., 0., 1., 1., 1.],\n",
              "        [0., 0., 0., 1., 1.],\n",
              "        [0., 0., 0., 0., 1.],\n",
              "        [0., 0., 0., 0., 0.]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J0YmBI89U286",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d11dc79e-91b1-405a-c600-05d386305fb9"
      },
      "source": [
        "# pad_mask 조회\n",
        "pad_mask = get_pad_mask(dec_tokens)\n",
        "pad_mask"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 1, 5), dtype=float32, numpy=array([[[0., 0., 0., 0., 1.]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8nUEj4W-0O8X",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bb479f27-ef7f-4377-8e16-8acd3b700599"
      },
      "source": [
        "# pad mask 또는 ahead mask인 부분은 모두 mask\n",
        "mask = tf.maximum(ahead_mask4, pad_mask)\n",
        "mask"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 5), dtype=float32, numpy=\n",
              "array([[[0., 1., 1., 1., 1.],\n",
              "        [0., 0., 1., 1., 1.],\n",
              "        [0., 0., 0., 1., 1.],\n",
              "        [0., 0., 0., 0., 1.],\n",
              "        [0., 0., 0., 0., 1.]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oZcLQR0NKINv",
        "colab_type": "text"
      },
      "source": [
        "### Function"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xQ0bR5TuzJRJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_ahead_mask(tokens, i_pad=0):\n",
        "    \"\"\"\n",
        "    ahead mask 계산하는 함수\n",
        "    :param tokens: tokens (bs, n_seq)\n",
        "    :param i_pad: id of pad\n",
        "    :return mask: ahead and pad mask (ahead or pad: 1, other: 0)\n",
        "    \"\"\"\n",
        "    n_seq = tf.shape(tokens)[1]\n",
        "    ahead_mask = 1 - tf.linalg.band_part(tf.ones((n_seq, n_seq)), -1, 0)\n",
        "    ahead_mask = tf.expand_dims(ahead_mask, axis=0)\n",
        "    pad_mask = get_pad_mask(tokens, i_pad)\n",
        "    mask = tf.maximum(ahead_mask, pad_mask)\n",
        "    return mask"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PTpTinUoKLc1",
        "colab_type": "text"
      },
      "source": [
        "### Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hAXZlBpqVJzT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bb4ca634-416b-4ded-d6e5-cfaf22d554b9"
      },
      "source": [
        "# 기능 확인\n",
        "dec_ahead_mask = get_ahead_mask(dec_tokens)\n",
        "# 함수로 구한 mask와 계산한 mask가 같으면 정상\n",
        "assert np.array_equal(mask, dec_ahead_mask)\n",
        "dec_tokens, dec_ahead_mask"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([[8, 7, 9, 1, 0]]), <tf.Tensor: shape=(1, 5, 5), dtype=float32, numpy=\n",
              " array([[[0., 1., 1., 1., 1.],\n",
              "         [0., 0., 1., 1., 1.],\n",
              "         [0., 0., 0., 1., 1.],\n",
              "         [0., 0., 0., 0., 1.],\n",
              "         [0., 0., 0., 0., 1.]]], dtype=float32)>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "wWAtDOgtLHZr"
      },
      "source": [
        "## Scaled dot product attention (Encoder Self Attention)\n",
        "\n",
        "![](https://paul-hyun.github.io/assets/transformer/scaled_dot_attention_encoder.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "owr3DtdjLHZs"
      },
      "source": [
        "### Tutorial"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "amQb_h2Kw_Fn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "abd09f80-6819-4498-f5a4-2fb01a443fb0"
      },
      "source": [
        "weights = np.random.randint(1, 10, (config.n_vocab, 4)) / 10\n",
        "weights"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.1, 0.7, 0.3, 0.1],\n",
              "       [0.6, 0.3, 0.7, 0.4],\n",
              "       [0.8, 0.1, 0.1, 0.4],\n",
              "       [0.3, 0.4, 0.2, 0.4],\n",
              "       [0.2, 0.4, 0.8, 0.2],\n",
              "       [0.8, 0.5, 0.1, 0.6],\n",
              "       [0.2, 0.6, 0.5, 0.1],\n",
              "       [0.9, 0.9, 0.7, 0.9],\n",
              "       [0.7, 0.4, 0.2, 0.3],\n",
              "       [0.6, 0.3, 0.6, 0.7]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "tbWZpQAFLHZs",
        "colab": {}
      },
      "source": [
        "# embedding layer / 기능 확인을 위해 weights를 정수로 초기화 함\n",
        "int_embedding = tf.keras.layers.Embedding(input_dim=config.n_vocab, output_dim=4, weights=[weights])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "AVzEuM5XLHZ0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "386c995c-e5b8-470a-ed86-4b7c71ffb092"
      },
      "source": [
        "# enc_input pad mask\n",
        "enc_self_mask = get_pad_mask(enc_tokens)\n",
        "enc_self_mask"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 1, 6), dtype=float32, numpy=array([[[0., 0., 0., 0., 0., 1.]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "VMLQN5_oLHZy",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e36380ae-2b74-4a24-f937-8627c53a6aef"
      },
      "source": [
        "# Q value\n",
        "Q = int_embedding(enc_tokens)\n",
        "Q"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 4), dtype=float32, numpy=\n",
              "array([[[0.2, 0.4, 0.8, 0.2],\n",
              "        [0.9, 0.9, 0.7, 0.9],\n",
              "        [0.2, 0.6, 0.5, 0.1],\n",
              "        [0.8, 0.5, 0.1, 0.6],\n",
              "        [0.6, 0.3, 0.6, 0.7],\n",
              "        [0.1, 0.7, 0.3, 0.1]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "RwFCUjukLHZv",
        "colab": {}
      },
      "source": [
        "# Q, K, V 모두 동일한 값\n",
        "K = Q\n",
        "V = K"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "usRoSZ1ZLHZ2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4ec27919-7360-4de8-8c6c-114647dde7c4"
      },
      "source": [
        "# attention score\n",
        "attention_score = tf.matmul(Q, K, transpose_b=True)\n",
        "attention_score"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 6), dtype=float32, numpy=\n",
              "array([[[0.88000005, 1.2799999 , 0.7       , 0.56000006, 0.86      ,\n",
              "         0.56      ],\n",
              "        [1.2799999 , 2.9199998 , 1.1600001 , 1.78      , 1.86      ,\n",
              "         1.02      ],\n",
              "        [0.7       , 1.1600001 , 0.65999997, 0.57000005, 0.67      ,\n",
              "         0.6       ],\n",
              "        [0.56000006, 1.78      , 0.57000005, 1.26      , 1.11      ,\n",
              "         0.52      ],\n",
              "        [0.86      , 1.86      , 0.67      , 1.11      , 1.3000001 ,\n",
              "         0.52000004],\n",
              "        [0.56      , 1.02      , 0.6       , 0.52      , 0.52000004,\n",
              "         0.59999996]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "zi_Kr39mLHZ3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f8e6fd34-f242-426c-a91c-09779c9303ce"
      },
      "source": [
        "# scale 계산\n",
        "scale = tf.math.sqrt(tf.cast(tf.shape(K)[-1], tf.float32))\n",
        "scale"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(), dtype=float32, numpy=2.0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "tNfrVcGzLHZ5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5768631a-0a6e-45db-f160-a5959180c736"
      },
      "source": [
        "# attention_score를 scale로 나눔\n",
        "attention_scaled = tf.math.divide(attention_score, scale)\n",
        "attention_scaled"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 6), dtype=float32, numpy=\n",
              "array([[[0.44000003, 0.6399999 , 0.35      , 0.28000003, 0.43      ,\n",
              "         0.28      ],\n",
              "        [0.6399999 , 1.4599999 , 0.58000004, 0.89      , 0.93      ,\n",
              "         0.51      ],\n",
              "        [0.35      , 0.58000004, 0.32999998, 0.28500003, 0.335     ,\n",
              "         0.3       ],\n",
              "        [0.28000003, 0.89      , 0.28500003, 0.63      , 0.555     ,\n",
              "         0.26      ],\n",
              "        [0.43      , 0.93      , 0.335     , 0.555     , 0.65000004,\n",
              "         0.26000002],\n",
              "        [0.28      , 0.51      , 0.3       , 0.26      , 0.26000002,\n",
              "         0.29999998]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "x8eR3vsDLHZ-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "56221d4c-ff18-4cf0-8c2c-3f79dd34c2ff"
      },
      "source": [
        "# mask 해야할 부분이 매우 큰 음수를 뺌\n",
        "attention_masked = attention_scaled - 1.e9 * enc_self_mask\n",
        "attention_masked"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 6), dtype=float32, numpy=\n",
              "array([[[ 4.4000003e-01,  6.3999993e-01,  3.4999999e-01,  2.8000003e-01,\n",
              "          4.3000001e-01, -1.0000000e+09],\n",
              "        [ 6.3999993e-01,  1.4599999e+00,  5.8000004e-01,  8.8999999e-01,\n",
              "          9.3000001e-01, -1.0000000e+09],\n",
              "        [ 3.4999999e-01,  5.8000004e-01,  3.2999998e-01,  2.8500003e-01,\n",
              "          3.3500001e-01, -1.0000000e+09],\n",
              "        [ 2.8000003e-01,  8.8999999e-01,  2.8500003e-01,  6.3000000e-01,\n",
              "          5.5500001e-01, -1.0000000e+09],\n",
              "        [ 4.3000001e-01,  9.3000001e-01,  3.3500001e-01,  5.5500001e-01,\n",
              "          6.5000004e-01, -1.0000000e+09],\n",
              "        [ 2.8000000e-01,  5.0999999e-01,  3.0000001e-01,  2.5999999e-01,\n",
              "          2.6000002e-01, -1.0000000e+09]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "K8XQ3oVsLHZ_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "919c13d9-c958-4b87-80da-8cc424d2019b"
      },
      "source": [
        "# attention 확률분포 계산\n",
        "attention_prob = tf.nn.softmax(attention_masked, axis=-1)\n",
        "attention_prob"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 6), dtype=float32, numpy=\n",
              "array([[[0.20090295, 0.2453834 , 0.18361147, 0.17119819, 0.19890393,\n",
              "         0.        ],\n",
              "        [0.14635466, 0.33229822, 0.13783163, 0.18792309, 0.1955924 ,\n",
              "         0.        ],\n",
              "        [0.19376172, 0.24386853, 0.18992499, 0.18156782, 0.19087699,\n",
              "         0.        ],\n",
              "        [0.15195161, 0.2796565 , 0.15271327, 0.21562959, 0.20004895,\n",
              "         0.        ],\n",
              "        [0.16842617, 0.27768782, 0.1531622 , 0.19085185, 0.20987193,\n",
              "         0.        ],\n",
              "        [0.19086887, 0.24022757, 0.19472468, 0.18708941, 0.18708941,\n",
              "         0.        ]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "G5ooI4b0LHaB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d5237896-9e73-410f-83bf-d7eec18a90ee"
      },
      "source": [
        "# attention output 계산\n",
        "attention_output = tf.matmul(attention_prob, V)\n",
        "attention_output"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 4), dtype=float32, numpy=\n",
              "array([[[0.5540489 , 0.55664337, 0.56075865, 0.52133846],\n",
              "        [0.6235995 , 0.5929485 , 0.55475605, 0.59179103],\n",
              "        [0.5559995 , 0.5589884 , 0.5533628 , 0.5197811 ],\n",
              "        [0.6051569 , 0.5719289 , 0.5352698 , 0.56676453],\n",
              "        [0.5928414 , 0.5675743 , 0.55071187, 0.56034195],\n",
              "        [0.55524874, 0.5590587 , 0.5491793 , 0.5170673 ]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 57
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "UP9HqezULHaC"
      },
      "source": [
        "### Class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "gh5Ya0LwLHaC",
        "colab": {}
      },
      "source": [
        "class ScaleDotProductAttention(tf.keras.layers.Layer):\n",
        "    \"\"\"\n",
        "    Scale Dot Product Attention Class\n",
        "    \"\"\"\n",
        "    def __init__(self, name=\"scale_dot_product_attention\"):\n",
        "        \"\"\"\n",
        "        생성자\n",
        "        :param name: layer name\n",
        "        \"\"\"\n",
        "        super().__init__(name=name)\n",
        "\n",
        "    def call(self, Q, K, V, attn_mask):\n",
        "        \"\"\"\n",
        "        layer 실행\n",
        "        :param Q: Q value\n",
        "        :param K: K value\n",
        "        :param V: V value\n",
        "        :param attn_mask: 실행 모드\n",
        "        :return attn_out: attention 실행 결과\n",
        "        \"\"\"\n",
        "        attn_score = tf.matmul(Q, K, transpose_b=True)\n",
        "        scale = tf.math.sqrt(tf.cast(tf.shape(K)[-1], tf.float32))\n",
        "        attn_scale = tf.math.divide(attn_score, scale)\n",
        "        attn_scale -= 1.e9 * attn_mask\n",
        "        attn_prob = tf.nn.softmax(attn_scale, axis=-1)\n",
        "        attn_out = tf.matmul(attn_prob, V)\n",
        "        return attn_out"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "QJ4wDtrPLHaE"
      },
      "source": [
        "### Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "uzMyTIFrLHaE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "89cca6cf-b86c-47b8-bf7f-8822eb9b49a8"
      },
      "source": [
        "# ScaleDotProductAttention 클래스 기능 확인\n",
        "attention = ScaleDotProductAttention()\n",
        "\n",
        "attn_out = attention(Q, K, V, enc_self_mask)\n",
        "# class 값과 직접 계산한 attention_output과 동일하면 정상\n",
        "assert np.array_equal(attention_output, attn_out)\n",
        "attn_out"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 4), dtype=float32, numpy=\n",
              "array([[[0.5540489 , 0.55664337, 0.56075865, 0.52133846],\n",
              "        [0.6235995 , 0.5929485 , 0.55475605, 0.59179103],\n",
              "        [0.5559995 , 0.5589884 , 0.5533628 , 0.5197811 ],\n",
              "        [0.6051569 , 0.5719289 , 0.5352698 , 0.56676453],\n",
              "        [0.5928414 , 0.5675743 , 0.55071187, 0.56034195],\n",
              "        [0.55524874, 0.5590587 , 0.5491793 , 0.5170673 ]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZZYToSV7yRVF",
        "colab_type": "text"
      },
      "source": [
        "## Scaled dot product attention (Decoder Masked Self Attentio)\n",
        "\n",
        "![](https://paul-hyun.github.io/assets/transformer/scaled_dot_attention_decoder.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VlhvQ5TcKlp_",
        "colab_type": "text"
      },
      "source": [
        "### Tutorial"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t2vVkvAQ099L",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f6e9ce6f-fa46-4ee6-dbeb-21bf3943fcd0"
      },
      "source": [
        "# ahead mask 계산\n",
        "dec_self_mask = get_ahead_mask(dec_tokens)\n",
        "dec_self_mask"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 5), dtype=float32, numpy=\n",
              "array([[[0., 1., 1., 1., 1.],\n",
              "        [0., 0., 1., 1., 1.],\n",
              "        [0., 0., 0., 1., 1.],\n",
              "        [0., 0., 0., 0., 1.],\n",
              "        [0., 0., 0., 0., 1.]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BHDRgT9sY9XN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "519b647e-d440-4da6-ab4a-cb20d0c838d7"
      },
      "source": [
        "# Q\n",
        "Q = int_embedding(dec_tokens)\n",
        "Q"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 4), dtype=float32, numpy=\n",
              "array([[[0.7, 0.4, 0.2, 0.3],\n",
              "        [0.9, 0.9, 0.7, 0.9],\n",
              "        [0.6, 0.3, 0.6, 0.7],\n",
              "        [0.6, 0.3, 0.7, 0.4],\n",
              "        [0.1, 0.7, 0.3, 0.1]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "68Vh6DMK2rUH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Q, K, V는 모두 같음\n",
        "K = Q\n",
        "V = K"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6VdnLr0k2Q0t",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a28dde34-47c6-416c-dc69-b4bfc23b784f"
      },
      "source": [
        "# attention score 계산\n",
        "attention_score = tf.matmul(Q, K, transpose_b=True)\n",
        "attention_score"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 5), dtype=float32, numpy=\n",
              "array([[[0.78      , 1.4       , 0.87      , 0.8       , 0.44      ],\n",
              "        [1.4       , 2.9199998 , 1.86      , 1.6600001 , 1.02      ],\n",
              "        [0.87      , 1.86      , 1.3000001 , 1.15      , 0.52000004],\n",
              "        [0.8       , 1.6600001 , 1.15      , 1.1       , 0.52000004],\n",
              "        [0.44      , 1.02      , 0.52000004, 0.52000004, 0.59999996]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 63
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "84tkL_dr2V21",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "50fa1609-98c2-4c35-8c24-38f33ceef479"
      },
      "source": [
        "# attention score를 scal로 나눔\n",
        "attention_scaled = tf.math.divide(attention_score, scale)\n",
        "attention_scaled"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 5), dtype=float32, numpy=\n",
              "array([[[0.39      , 0.7       , 0.435     , 0.4       , 0.22      ],\n",
              "        [0.7       , 1.4599999 , 0.93      , 0.83000004, 0.51      ],\n",
              "        [0.435     , 0.93      , 0.65000004, 0.575     , 0.26000002],\n",
              "        [0.4       , 0.83000004, 0.575     , 0.55      , 0.26000002],\n",
              "        [0.22      , 0.51      , 0.26000002, 0.26000002, 0.29999998]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dBK4CDoZ2ZR2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c7d747b9-44b9-4365-dc80-1124bcdf2846"
      },
      "source": [
        "# mask 해야할 부분이 매우 큰 음수를 뺌\n",
        "attention_masked = attention_scaled - 1.e9 * dec_self_mask\n",
        "attention_masked"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 5), dtype=float32, numpy=\n",
              "array([[[ 3.8999999e-01, -1.0000000e+09, -1.0000000e+09, -1.0000000e+09,\n",
              "         -1.0000000e+09],\n",
              "        [ 6.9999999e-01,  1.4599999e+00, -1.0000000e+09, -1.0000000e+09,\n",
              "         -1.0000000e+09],\n",
              "        [ 4.3500000e-01,  9.3000001e-01,  6.5000004e-01, -1.0000000e+09,\n",
              "         -1.0000000e+09],\n",
              "        [ 4.0000001e-01,  8.3000004e-01,  5.7499999e-01,  5.5000001e-01,\n",
              "         -1.0000000e+09],\n",
              "        [ 2.2000000e-01,  5.0999999e-01,  2.6000002e-01,  2.6000002e-01,\n",
              "         -1.0000000e+09]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 65
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TmBbfOfq2dnU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9e4db982-ff08-484a-e550-a7a69dee2b16"
      },
      "source": [
        "# attention 확률분포 계산\n",
        "attention_prob = tf.nn.softmax(attention_masked, axis=-1)\n",
        "attention_prob"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 5), dtype=float32, numpy=\n",
              "array([[[1.        , 0.        , 0.        , 0.        , 0.        ],\n",
              "        [0.31864628, 0.68135375, 0.        , 0.        , 0.        ],\n",
              "        [0.25770804, 0.42276955, 0.31952235, 0.        , 0.        ],\n",
              "        [0.20448484, 0.31434587, 0.24359179, 0.23757747, 0.        ],\n",
              "        [0.22634424, 0.30249268, 0.23558153, 0.23558153, 0.        ]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 66
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5n9pOBvR3NkG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cfdbf8f2-e9be-4129-a496-1d6d4b9e811a"
      },
      "source": [
        "# attention output 계산\n",
        "attention_output = tf.matmul(attention_prob, V)\n",
        "attention_output"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 4), dtype=float32, numpy=\n",
              "array([[[0.7       , 0.4       , 0.2       , 0.3       ],\n",
              "        [0.83627075, 0.7406769 , 0.5406769 , 0.70881224],\n",
              "        [0.7526016 , 0.5794325 , 0.5391937 , 0.68147063],\n",
              "        [0.7147522 , 0.509056  , 0.57339835, 0.60980195],\n",
              "        [0.71338224, 0.50413   , 0.56326973, 0.5992864 ]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 67
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O33u8_SWKrH4",
        "colab_type": "text"
      },
      "source": [
        "### Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FeBoulyk2Ij3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f2170988-ed44-4e83-cfdb-20b1d24b41e1"
      },
      "source": [
        "# ScaleDotProductAttention 클래스 기능 확인\n",
        "attention = ScaleDotProductAttention()\n",
        "\n",
        "attn_out = attention(Q, K, V, dec_self_mask)\n",
        "# class 값과 직접 계산한 attention_output과 동일하면 정상\n",
        "assert np.array_equal(attention_output, attn_out)\n",
        "attn_out"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 4), dtype=float32, numpy=\n",
              "array([[[0.7       , 0.4       , 0.2       , 0.3       ],\n",
              "        [0.83627075, 0.7406769 , 0.5406769 , 0.70881224],\n",
              "        [0.7526016 , 0.5794325 , 0.5391937 , 0.68147063],\n",
              "        [0.7147522 , 0.509056  , 0.57339835, 0.60980195],\n",
              "        [0.71338224, 0.50413   , 0.56326973, 0.5992864 ]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 68
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oPBEA8FAl39-",
        "colab_type": "text"
      },
      "source": [
        "## Scaled dot product attention (Encoder Decoder Attention)\n",
        "\n",
        "![](https://paul-hyun.github.io/assets/transformer/scaled_dot_attention_ende.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "64BhMxZ4KUI1",
        "colab_type": "text"
      },
      "source": [
        "### Tutorial"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UZMU3cCBplu5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1bfa9971-a977-4264-e2b2-08e49f849b96"
      },
      "source": [
        "# K pad mask\n",
        "ende_attn_mask = get_pad_mask(enc_tokens)\n",
        "ende_attn_mask"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 1, 6), dtype=float32, numpy=array([[[0., 0., 0., 0., 0., 1.]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P6Y713Sw59_3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a334caa8-b3b7-4147-c108-7c7a6d7a0a5a"
      },
      "source": [
        "Q = int_embedding(dec_tokens)\n",
        "Q"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 4), dtype=float32, numpy=\n",
              "array([[[0.7, 0.4, 0.2, 0.3],\n",
              "        [0.9, 0.9, 0.7, 0.9],\n",
              "        [0.6, 0.3, 0.6, 0.7],\n",
              "        [0.6, 0.3, 0.7, 0.4],\n",
              "        [0.1, 0.7, 0.3, 0.1]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l4Zm-xaC5l13",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c2c8de88-ee32-4398-8147-5c17c703a16a"
      },
      "source": [
        "K = int_embedding(enc_tokens)\n",
        "V = K\n",
        "K"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 4), dtype=float32, numpy=\n",
              "array([[[0.2, 0.4, 0.8, 0.2],\n",
              "        [0.9, 0.9, 0.7, 0.9],\n",
              "        [0.2, 0.6, 0.5, 0.1],\n",
              "        [0.8, 0.5, 0.1, 0.6],\n",
              "        [0.6, 0.3, 0.6, 0.7],\n",
              "        [0.1, 0.7, 0.3, 0.1]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 71
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5Xfr2fWUpu2M",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ebdc3205-3cce-409b-984c-7434ff2a68cf"
      },
      "source": [
        "# attention score\n",
        "attention_score = tf.matmul(Q, K, transpose_b=True)\n",
        "attention_score"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 6), dtype=float32, numpy=\n",
              "array([[[0.52000004, 1.4       , 0.51      , 0.96      , 0.87      ,\n",
              "         0.44      ],\n",
              "        [1.2799999 , 2.9199998 , 1.1600001 , 1.78      , 1.86      ,\n",
              "         1.02      ],\n",
              "        [0.86      , 1.86      , 0.67      , 1.11      , 1.3000001 ,\n",
              "         0.52000004],\n",
              "        [0.88      , 1.6600001 , 0.69      , 0.94      , 1.15      ,\n",
              "         0.52000004],\n",
              "        [0.56      , 1.02      , 0.6       , 0.52      , 0.52000004,\n",
              "         0.59999996]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-wCXHWVgp2KI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9a2d2fcb-be7c-4f20-fe41-17d086650362"
      },
      "source": [
        "# scale 계산\n",
        "scale = tf.math.sqrt(tf.cast(tf.shape(K)[-1], tf.float32))\n",
        "scale"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(), dtype=float32, numpy=2.0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 73
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yz1ULURnqCQg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5fb0472a-0cde-4a19-ad5c-4e64c5863c69"
      },
      "source": [
        "# attention_score를 scale로 나눔\n",
        "attention_scaled = tf.math.divide(attention_score, scale)\n",
        "attention_scaled"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 6), dtype=float32, numpy=\n",
              "array([[[0.26000002, 0.7       , 0.255     , 0.48      , 0.435     ,\n",
              "         0.22      ],\n",
              "        [0.6399999 , 1.4599999 , 0.58000004, 0.89      , 0.93      ,\n",
              "         0.51      ],\n",
              "        [0.43      , 0.93      , 0.335     , 0.555     , 0.65000004,\n",
              "         0.26000002],\n",
              "        [0.44      , 0.83000004, 0.345     , 0.47      , 0.575     ,\n",
              "         0.26000002],\n",
              "        [0.28      , 0.51      , 0.3       , 0.26      , 0.26000002,\n",
              "         0.29999998]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 74
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8sn-fgrBsKdv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b9674442-9104-4435-8875-c20b4ba44100"
      },
      "source": [
        "# mask 해야할 부분이 매우 큰 음수를 뺌\n",
        "attention_masked = attention_scaled - 1.e9 * ende_attn_mask\n",
        "attention_masked"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 6), dtype=float32, numpy=\n",
              "array([[[ 2.6000002e-01,  6.9999999e-01,  2.5500000e-01,  4.7999999e-01,\n",
              "          4.3500000e-01, -1.0000000e+09],\n",
              "        [ 6.3999993e-01,  1.4599999e+00,  5.8000004e-01,  8.8999999e-01,\n",
              "          9.3000001e-01, -1.0000000e+09],\n",
              "        [ 4.3000001e-01,  9.3000001e-01,  3.3500001e-01,  5.5500001e-01,\n",
              "          6.5000004e-01, -1.0000000e+09],\n",
              "        [ 4.4000000e-01,  8.3000004e-01,  3.4500000e-01,  4.7000000e-01,\n",
              "          5.7499999e-01, -1.0000000e+09],\n",
              "        [ 2.8000000e-01,  5.0999999e-01,  3.0000001e-01,  2.5999999e-01,\n",
              "          2.6000002e-01, -1.0000000e+09]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 75
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZjKnrJIYsabT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d61a3d3a-6e9c-4f25-e00c-2a30f95da9ea"
      },
      "source": [
        "# attention 확률분포 계산\n",
        "attention_prob = tf.nn.softmax(attention_masked, axis=-1)\n",
        "attention_prob"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 6), dtype=float32, numpy=\n",
              "array([[[0.16708319, 0.25943127, 0.16624986, 0.20819847, 0.19903722,\n",
              "         0.        ],\n",
              "        [0.14635466, 0.33229822, 0.13783163, 0.18792309, 0.1955924 ,\n",
              "         0.        ],\n",
              "        [0.16842617, 0.27768782, 0.1531622 , 0.19085185, 0.20987193,\n",
              "         0.        ],\n",
              "        [0.17981261, 0.2655798 , 0.16351672, 0.18528873, 0.20580216,\n",
              "         0.        ],\n",
              "        [0.19086887, 0.24022757, 0.19472468, 0.18708941, 0.18708941,\n",
              "         0.        ]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 76
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dkYVG5hxsjel",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d2b7ed1e-c548-4c01-ee39-fcd451659a2e"
      },
      "source": [
        "# attention output 계산\n",
        "attention_output = tf.matmul(attention_prob, V)\n",
        "attention_output"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 4), dtype=float32, numpy=\n",
              "array([[[0.58613586, 0.56388175, 0.5386356 , 0.5477749 ],\n",
              "        [0.6235995 , 0.5929485 , 0.55475605, 0.59179103],\n",
              "        [0.5928414 , 0.5675743 , 0.55071187, 0.56034195],\n",
              "        [0.57939994, 0.5634419 , 0.55352443, 0.5465708 ],\n",
              "        [0.55524874, 0.5590587 , 0.5491793 , 0.5170673 ]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 77
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SbQ-4OZ2KdbH",
        "colab_type": "text"
      },
      "source": [
        "### Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "41x0arlEwnFJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0e03e1e0-dc59-4db8-a735-b511e8c12d2b"
      },
      "source": [
        "# ScaleDotProductAttention 클래스 기능 확인\n",
        "attention = ScaleDotProductAttention()\n",
        "\n",
        "attn_out = attention(Q, K, V, ende_attn_mask)\n",
        "# class 값과 직접 계산한 attention_output과 동일하면 정상\n",
        "assert np.array_equal(attention_output, attn_out)\n",
        "attn_out"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 4), dtype=float32, numpy=\n",
              "array([[[0.58613586, 0.56388175, 0.5386356 , 0.5477749 ],\n",
              "        [0.6235995 , 0.5929485 , 0.55475605, 0.59179103],\n",
              "        [0.5928414 , 0.5675743 , 0.55071187, 0.56034195],\n",
              "        [0.57939994, 0.5634419 , 0.55352443, 0.5465708 ],\n",
              "        [0.55524874, 0.5590587 , 0.5491793 , 0.5170673 ]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 78
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sc0xdcL-6ZZl",
        "colab_type": "text"
      },
      "source": [
        "## Multi Head Attention\n",
        "\n",
        "![](https://paul-hyun.github.io/assets/transformer/multihead_attention.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W_ks6mcOKzow",
        "colab_type": "text"
      },
      "source": [
        "### Tutorial"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VGCkd7u65dBq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# SharedEmbedding\n",
        "embedding = SharedEmbedding(config)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tgRVy_TacLgU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ae32d4ea-add6-4fe4-a97e-ca50a31d340e"
      },
      "source": [
        "# Q\n",
        "Q = embedding(dec_tokens)\n",
        "Q"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 8), dtype=float32, numpy=\n",
              "array([[[-1.8684977 , -0.10887106, -1.6283252 , -0.09412991,\n",
              "         -0.9135378 , -0.5230187 ,  0.18054976,  1.0659311 ],\n",
              "        [-1.851637  , -0.52205473, -1.1127012 ,  0.556745  ,\n",
              "         -1.2216276 ,  0.70823485, -0.7989583 , -0.48866653],\n",
              "        [-0.20390666,  0.557556  , -0.40127853,  0.23037532,\n",
              "          0.02648634, -1.6108699 , -0.32580003, -0.3013221 ],\n",
              "        [ 0.8284642 ,  1.1036007 , -0.78627795,  1.6918567 ,\n",
              "         -0.34767905,  1.3073093 ,  0.2444622 , -0.7029787 ],\n",
              "        [-0.3198804 ,  1.3575249 , -0.15776914,  0.5804112 ,\n",
              "          0.4537885 ,  0.9504126 ,  1.6937656 ,  1.2484983 ]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 80
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5UlS_GDGCchP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "597def86-190b-4e0e-ac52-401f0f4b7e13"
      },
      "source": [
        "# K, V\n",
        "K = embedding(enc_tokens)\n",
        "V = K\n",
        "K"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 8), dtype=float32, numpy=\n",
              "array([[[ 0.19626832,  1.6160245 , -1.5125287 ,  0.8802097 ,\n",
              "          0.43964735, -0.49681696,  0.9233131 , -1.0172113 ],\n",
              "        [-1.851637  , -0.52205473, -1.1127012 ,  0.556745  ,\n",
              "         -1.2216276 ,  0.70823485, -0.7989583 , -0.48866653],\n",
              "        [-0.39263427, -1.7194668 ,  1.8227654 , -0.15567164,\n",
              "         -0.6654087 ,  0.68538827,  0.68061614,  0.74703187],\n",
              "        [-1.3610662 , -1.8012775 , -0.09980725, -0.69201565,\n",
              "         -1.6705056 , -0.07607771, -0.5028183 , -0.22716261],\n",
              "        [-0.20390666,  0.557556  , -0.40127853,  0.23037532,\n",
              "          0.02648634, -1.6108699 , -0.32580003, -0.3013221 ],\n",
              "        [-0.3198804 ,  1.3575249 , -0.15776914,  0.5804112 ,\n",
              "          0.4537885 ,  0.9504126 ,  1.6937656 ,  1.2484983 ]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 81
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XcgGGASr-XKZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fcc7f549-39a2-49a0-8aa5-c1b9f9b400ae"
      },
      "source": [
        "# Q에 Weight W_Q 곱함\n",
        "W_Q = tf.keras.layers.Dense(config.n_head * config.d_head)\n",
        "Q_multi_1 = W_Q(Q)\n",
        "Q_multi_1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 8), dtype=float32, numpy=\n",
              "array([[[ 0.60726416,  0.36179903,  0.32159328,  1.0295479 ,\n",
              "          2.0804245 , -0.7570441 , -0.34533682,  1.0343486 ],\n",
              "        [ 0.46542645,  0.6619142 , -0.23965865, -0.07134926,\n",
              "          1.7227607 ,  0.51219004, -0.8823885 ,  0.36687952],\n",
              "        [ 0.541306  , -0.90740347,  0.27299052, -0.4724496 ,\n",
              "          0.66669774, -0.36000645,  0.40457246,  0.09094185],\n",
              "        [ 0.7040179 ,  1.3128388 ,  0.02026618, -0.20661888,\n",
              "          1.2101818 ,  0.39174786, -0.87256515,  2.276158  ],\n",
              "        [-0.39734793,  1.7864552 , -0.09988433,  0.5383763 ,\n",
              "          0.74667156, -0.8564163 ,  0.32024744,  2.4470553 ]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 82
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bzHKYhM7-rzx",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7eb8002f-46a0-4c74-c20f-91565311307c"
      },
      "source": [
        "# 여러 개의 head로 분할\n",
        "Q_multi_2 = tf.reshape(Q_multi_1, [tf.shape(Q)[0], -1, config.n_head, config.d_head])  # (bs, n_seq, n_head, d_head)\n",
        "Q_multi_2"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 2, 4), dtype=float32, numpy=\n",
              "array([[[[ 0.60726416,  0.36179903,  0.32159328,  1.0295479 ],\n",
              "         [ 2.0804245 , -0.7570441 , -0.34533682,  1.0343486 ]],\n",
              "\n",
              "        [[ 0.46542645,  0.6619142 , -0.23965865, -0.07134926],\n",
              "         [ 1.7227607 ,  0.51219004, -0.8823885 ,  0.36687952]],\n",
              "\n",
              "        [[ 0.541306  , -0.90740347,  0.27299052, -0.4724496 ],\n",
              "         [ 0.66669774, -0.36000645,  0.40457246,  0.09094185]],\n",
              "\n",
              "        [[ 0.7040179 ,  1.3128388 ,  0.02026618, -0.20661888],\n",
              "         [ 1.2101818 ,  0.39174786, -0.87256515,  2.276158  ]],\n",
              "\n",
              "        [[-0.39734793,  1.7864552 , -0.09988433,  0.5383763 ],\n",
              "         [ 0.74667156, -0.8564163 ,  0.32024744,  2.4470553 ]]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 83
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vdCKZWoV-7Ad",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "93375320-4a2f-4b2d-d9e4-25868201b4df"
      },
      "source": [
        "# bs, number of head, seq length, dim of head 가 되도록 정렬\n",
        "Q_multi = tf.transpose(Q_multi_2, [0, 2, 1, 3])  # (bs, h_head, n_seq, d_head)\n",
        "Q_multi"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 2, 5, 4), dtype=float32, numpy=\n",
              "array([[[[ 0.60726416,  0.36179903,  0.32159328,  1.0295479 ],\n",
              "         [ 0.46542645,  0.6619142 , -0.23965865, -0.07134926],\n",
              "         [ 0.541306  , -0.90740347,  0.27299052, -0.4724496 ],\n",
              "         [ 0.7040179 ,  1.3128388 ,  0.02026618, -0.20661888],\n",
              "         [-0.39734793,  1.7864552 , -0.09988433,  0.5383763 ]],\n",
              "\n",
              "        [[ 2.0804245 , -0.7570441 , -0.34533682,  1.0343486 ],\n",
              "         [ 1.7227607 ,  0.51219004, -0.8823885 ,  0.36687952],\n",
              "         [ 0.66669774, -0.36000645,  0.40457246,  0.09094185],\n",
              "         [ 1.2101818 ,  0.39174786, -0.87256515,  2.276158  ],\n",
              "         [ 0.74667156, -0.8564163 ,  0.32024744,  2.4470553 ]]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 84
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0g9OJ_Bk_OmD",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4bfa585f-9a10-40b3-90c3-5d48a5b53840"
      },
      "source": [
        "# K multi head로 분할\n",
        "W_K = tf.keras.layers.Dense(config.n_head * config.d_head)\n",
        "K_multi = tf.transpose(tf.reshape(W_K(K), [tf.shape(K)[0], -1, config.n_head, config.d_head]), [0, 2, 1, 3])  # (bs, h_head, n_seq, d_head)\n",
        "K_multi"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 2, 6, 4), dtype=float32, numpy=\n",
              "array([[[[ 0.44157654,  0.6153864 ,  1.0823402 , -0.86922544],\n",
              "         [-0.07734162, -1.2439793 , -0.50609183,  0.7221384 ],\n",
              "         [-0.3402935 , -0.2853378 , -1.0854981 ,  1.3541911 ],\n",
              "         [-0.669067  , -0.8918519 , -1.4189062 ,  0.19565569],\n",
              "         [-0.09156863,  0.6330453 ,  0.34754407, -0.74757963],\n",
              "         [ 1.2972405 ,  0.22881103,  0.8145461 ,  1.3349972 ]],\n",
              "\n",
              "        [[ 1.8166773 , -1.2446976 ,  0.44066074, -0.8677362 ],\n",
              "         [ 1.7250284 ,  0.31310377,  1.3237873 , -1.119676  ],\n",
              "         [-0.23594195,  2.147542  , -0.1751713 ,  0.52035725],\n",
              "         [ 0.3804278 ,  0.8608245 ,  0.5974173 , -0.5911523 ],\n",
              "         [-0.3509707 , -1.2586634 , -0.36838096, -0.14807178],\n",
              "         [ 1.167274  ,  0.708056  ,  0.5896891 , -0.9456644 ]]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 85
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JdUJuxmm_ZHp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "85f9b2ea-dc3f-4b8e-fe1e-b014bb8fa305"
      },
      "source": [
        "# V multi head로 분할\n",
        "W_V = tf.keras.layers.Dense(config.n_head * config.d_head)\n",
        "V_multi = tf.transpose(tf.reshape(W_V(V), [tf.shape(V)[0], -1, config.n_head, config.d_head]), [0, 2, 1, 3])  # (bs, h_head, n_seq, d_head)\n",
        "V_multi"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 2, 6, 4), dtype=float32, numpy=\n",
              "array([[[[-0.05052549,  1.3679097 ,  0.06428897, -1.794766  ],\n",
              "         [ 0.30957353, -1.4764541 , -0.75112426,  0.45604098],\n",
              "         [-0.8646058 , -2.518207  ,  0.688274  ,  2.2011814 ],\n",
              "         [-0.35476363, -1.8334684 , -0.7684364 ,  1.4162139 ],\n",
              "         [-0.6195556 ,  1.0833172 , -0.48860148, -1.2713012 ],\n",
              "         [ 0.90829015, -0.8477763 ,  2.126217  ,  0.9595523 ]],\n",
              "\n",
              "        [[-1.1524616 , -0.30813012, -0.50484705, -0.3681637 ],\n",
              "         [-0.10167963,  1.8378878 , -0.3448023 ,  0.42790192],\n",
              "         [-0.231125  ,  0.55959207, -0.55778337, -0.05705123],\n",
              "         [ 0.04607373,  1.2424754 ,  0.4524849 ,  1.4593087 ],\n",
              "         [-0.7989114 , -0.7697486 ,  0.7987878 ,  0.4939655 ],\n",
              "         [-0.48566872, -0.09066993, -1.7201098 , -0.80178815]]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 86
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qwMiDvqkB4eI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cb5a8791-b5a8-4bee-b2b3-bdbabbb68c64"
      },
      "source": [
        "# pad mask 구함 / decoder 일 경우는 ahead mask 구하면 됨\n",
        "ende_mask = get_pad_mask(enc_tokens)\n",
        "ende_mask"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 1, 6), dtype=float32, numpy=array([[[0., 0., 0., 0., 0., 1.]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 87
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WUYOdq1QCA0x",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "11475654-03af-46c2-9358-4c97481bb230"
      },
      "source": [
        "# Q_multi, K_mulit, V_multi와 동일한 차원이 되도록 차원을 늘려 줌\n",
        "ende_mask_multi = tf.expand_dims(ende_mask, axis=1)\n",
        "ende_mask_multi"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 1, 1, 6), dtype=float32, numpy=array([[[[0., 0., 0., 0., 0., 1.]]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 88
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yuQjJWfcCH44",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "eaffcbea-7380-4439-da3c-7bb9b1a89285"
      },
      "source": [
        "# ScaleDotProductAttention을 여러 head 동시 실행 함\n",
        "attention = ScaleDotProductAttention()\n",
        "attn_val_multi = attention(Q_multi, K_multi, V_multi, ende_mask_multi)  # (bs, n_head, Q_len, d_head)\n",
        "attn_val_multi"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 2, 5, 4), dtype=float32, numpy=\n",
              "array([[[[-0.34562594, -0.8479199 , -0.11944495,  0.36608973],\n",
              "         [-0.3459013 , -0.35639045, -0.20108911, -0.07225244],\n",
              "         [-0.21888992, -0.54503214, -0.33133984,  0.01956929],\n",
              "         [-0.3329003 ,  0.2834197 , -0.16911803, -0.6665487 ],\n",
              "         [-0.4335849 , -0.3412676 , -0.10640375, -0.03630596]],\n",
              "\n",
              "        [[-0.77040124,  0.25091752, -0.27838963,  0.02456305],\n",
              "         [-0.4761507 ,  0.64109826, -0.23332982,  0.24423584],\n",
              "         [-0.54133904,  0.5510782 , -0.13543905,  0.28242606],\n",
              "         [-0.39123476,  0.46834347, -0.24057709,  0.1913853 ],\n",
              "         [-0.6347244 ,  0.1411059 , -0.01146939,  0.23022278]]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 89
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QMK74028ERaN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9e18e43f-ce9f-46eb-b051-58eadbe54a4f"
      },
      "source": [
        "attention_transpose = tf.transpose(attn_val_multi, perm=[0, 2, 1, 3])  # (bs, Q_len, n_head, d_head)\n",
        "attention_transpose"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 2, 4), dtype=float32, numpy=\n",
              "array([[[[-0.34562594, -0.8479199 , -0.11944495,  0.36608973],\n",
              "         [-0.77040124,  0.25091752, -0.27838963,  0.02456305]],\n",
              "\n",
              "        [[-0.3459013 , -0.35639045, -0.20108911, -0.07225244],\n",
              "         [-0.4761507 ,  0.64109826, -0.23332982,  0.24423584]],\n",
              "\n",
              "        [[-0.21888992, -0.54503214, -0.33133984,  0.01956929],\n",
              "         [-0.54133904,  0.5510782 , -0.13543905,  0.28242606]],\n",
              "\n",
              "        [[-0.3329003 ,  0.2834197 , -0.16911803, -0.6665487 ],\n",
              "         [-0.39123476,  0.46834347, -0.24057709,  0.1913853 ]],\n",
              "\n",
              "        [[-0.4335849 , -0.3412676 , -0.10640375, -0.03630596],\n",
              "         [-0.6347244 ,  0.1411059 , -0.01146939,  0.23022278]]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 90
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nVO0kq3_EgG2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3442c9d7-3fac-4c99-90e8-5ff711307de3"
      },
      "source": [
        "attention_concat = tf.reshape(attention_transpose, [tf.shape(Q)[0], -1, config.n_head * config.d_head])  # (bs, Q_len, n_head * d_head)\n",
        "attention_concat"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 8), dtype=float32, numpy=\n",
              "array([[[-0.34562594, -0.8479199 , -0.11944495,  0.36608973,\n",
              "         -0.77040124,  0.25091752, -0.27838963,  0.02456305],\n",
              "        [-0.3459013 , -0.35639045, -0.20108911, -0.07225244,\n",
              "         -0.4761507 ,  0.64109826, -0.23332982,  0.24423584],\n",
              "        [-0.21888992, -0.54503214, -0.33133984,  0.01956929,\n",
              "         -0.54133904,  0.5510782 , -0.13543905,  0.28242606],\n",
              "        [-0.3329003 ,  0.2834197 , -0.16911803, -0.6665487 ,\n",
              "         -0.39123476,  0.46834347, -0.24057709,  0.1913853 ],\n",
              "        [-0.4335849 , -0.3412676 , -0.10640375, -0.03630596,\n",
              "         -0.6347244 ,  0.1411059 , -0.01146939,  0.23022278]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 91
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0lbienE8EwMF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "48e9c8fa-f0cb-4856-bc49-e2aeefb30095"
      },
      "source": [
        "WO = tf.keras.layers.Dense(config.d_model)\n",
        "attention_output = WO(attention_concat)\n",
        "attention_output"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 8), dtype=float32, numpy=\n",
              "array([[[ 0.49050608, -0.8050246 , -0.45042843,  0.45777676,\n",
              "          0.06806503, -0.33055833,  0.11655104,  1.1064618 ],\n",
              "        [ 0.38006845, -0.673133  , -0.5220271 ,  0.21313778,\n",
              "          0.2849807 ,  0.0296824 , -0.01816981,  0.7601285 ],\n",
              "        [ 0.4035927 , -0.7793803 , -0.3197016 ,  0.25294086,\n",
              "          0.29103556, -0.05029402,  0.03342265,  0.80691427],\n",
              "        [ 0.1614267 , -0.28013894, -0.42321342, -0.3257166 ,\n",
              "          0.22512603,  0.12906909,  0.18890849, -0.07056844],\n",
              "        [ 0.26344997, -0.61582273, -0.15614234, -0.0834959 ,\n",
              "          0.0115939 , -0.32596976,  0.5276856 ,  0.5310106 ]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 92
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hXWYTm6JK9OR",
        "colab_type": "text"
      },
      "source": [
        "### Class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jjC2xmX4F5ay",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class MultiHeadAttention(tf.keras.layers.Layer):\n",
        "    \"\"\"\n",
        "    Multi Head Attention Class\n",
        "    \"\"\"\n",
        "    def __init__(self, config, name=\"multi_head_attention\"):\n",
        "        \"\"\"\n",
        "        생성자\n",
        "        :param config: Config 객체\n",
        "        :param name: layer name\n",
        "        \"\"\"\n",
        "        super().__init__(name=name)\n",
        "\n",
        "        self.d_model = config.d_model\n",
        "        self.n_head = config.n_head\n",
        "        self.d_head = config.d_head\n",
        "\n",
        "        # Q, K, V input dense layer\n",
        "        self.W_Q = tf.keras.layers.Dense(config.n_head * config.d_head)\n",
        "        self.W_K = tf.keras.layers.Dense(config.n_head * config.d_head)\n",
        "        self.W_V = tf.keras.layers.Dense(config.n_head * config.d_head)\n",
        "        # Scale Dot Product Attention class\n",
        "        self.attention = ScaleDotProductAttention(name=\"self_attention\")\n",
        "        # output dense layer\n",
        "        self.W_O = tf.keras.layers.Dense(config.d_model)\n",
        "\n",
        "    def call(self, Q, K, V, attn_mask):\n",
        "        \"\"\"\n",
        "        layer 실행\n",
        "        :param Q: Q value\n",
        "        :param K: K value\n",
        "        :param V: V value\n",
        "        :param attn_mask: 실행 모드\n",
        "        :return attn_out: attention 실행 결과\n",
        "        \"\"\"\n",
        "        # reshape Q, K, V, attn_mask\n",
        "        batch_size = tf.shape(Q)[0]\n",
        "        Q_m = tf.transpose(tf.reshape(self.W_Q(Q), [batch_size, -1, self.n_head, self.d_head]), [0, 2, 1, 3])  # (bs, n_head, Q_len, d_head)\n",
        "        K_m = tf.transpose(tf.reshape(self.W_K(K), [batch_size, -1, self.n_head, self.d_head]), [0, 2, 1, 3])  # (bs, n_head, K_len, d_head)\n",
        "        V_m = tf.transpose(tf.reshape(self.W_V(V), [batch_size, -1, self.n_head, self.d_head]), [0, 2, 1, 3])  # (bs, n_head, K_len, d_head)\n",
        "        attn_mask_m = tf.expand_dims(attn_mask, axis=1)\n",
        "        # Scale Dot Product Attention with multi head Q, K, V, attn_mask\n",
        "        attn_out = self.attention(Q_m, K_m, V_m, attn_mask_m)  # (bs, n_head, Q_len, d_head)\n",
        "        # transpose and liner\n",
        "        attn_out_m = tf.transpose(attn_out, perm=[0, 2, 1, 3])  # (bs, Q_len, n_head, d_head)\n",
        "        attn_out = tf.reshape(attn_out_m, [batch_size, -1, config.n_head * config.d_head])  # (bs, Q_len, d_model)\n",
        "        attn_out = self.W_O(attn_out) # (bs, Q_len, d_model)\n",
        "\n",
        "        return attn_out"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f9HRMhWKK-6I",
        "colab_type": "text"
      },
      "source": [
        "### Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T8ONc_AXG7Db",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "538fa0ad-f54d-4fa6-dd1d-c4773adc13d1"
      },
      "source": [
        "attention = MultiHeadAttention(config)\n",
        "\n",
        "attn_out = attention(Q, K, V, ende_mask)\n",
        "# Q와 output의 shape 같으면 정상 (W_Q, W_K, W_V, W_O 값이 랜덤하게 초기화 되므로 값은 달라 짐)\n",
        "assert Q.shape == attn_out.shape\n",
        "attn_out"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 8), dtype=float32, numpy=\n",
              "array([[[-1.7263755 ,  0.01100424, -0.14146975, -1.4741412 ,\n",
              "          0.6791463 ,  0.5476541 , -1.3935212 ,  1.6705613 ],\n",
              "        [-1.3784302 ,  0.7695726 , -0.45037347, -1.3775394 ,\n",
              "          0.87207174, -0.3568232 , -0.64005435,  0.96092784],\n",
              "        [-0.9750034 , -0.19607307, -0.03950721, -0.85755706,\n",
              "          0.25044265,  0.84326804, -1.0721036 ,  1.1410964 ],\n",
              "        [-0.5620067 ,  0.6079061 , -0.26716498, -0.5765987 ,\n",
              "          0.34476867, -0.15077245, -0.14507551,  0.39121914],\n",
              "        [-1.350141  ,  0.18068255, -0.08150241, -1.1048428 ,\n",
              "          0.5075928 ,  0.48607206, -0.92029136,  1.2991698 ]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 94
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z3Jc-KlSQDMG",
        "colab_type": "text"
      },
      "source": [
        "## Feed Forward\n",
        "\n",
        "![](https://paul-hyun.github.io/assets/transformer/feedforward.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OnCeiVe2NrNz",
        "colab_type": "text"
      },
      "source": [
        "### Tutorial"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Eq9g080D19oi",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dd2616bf-d99e-4a3b-8afd-83f45f4b605d"
      },
      "source": [
        "# sublayer output\n",
        "sublayer_output = embedding(enc_tokens)\n",
        "sublayer_output"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 8), dtype=float32, numpy=\n",
              "array([[[ 0.19626832,  1.6160245 , -1.5125287 ,  0.8802097 ,\n",
              "          0.43964735, -0.49681696,  0.9233131 , -1.0172113 ],\n",
              "        [-1.851637  , -0.52205473, -1.1127012 ,  0.556745  ,\n",
              "         -1.2216276 ,  0.70823485, -0.7989583 , -0.48866653],\n",
              "        [-0.39263427, -1.7194668 ,  1.8227654 , -0.15567164,\n",
              "         -0.6654087 ,  0.68538827,  0.68061614,  0.74703187],\n",
              "        [-1.3610662 , -1.8012775 , -0.09980725, -0.69201565,\n",
              "         -1.6705056 , -0.07607771, -0.5028183 , -0.22716261],\n",
              "        [-0.20390666,  0.557556  , -0.40127853,  0.23037532,\n",
              "          0.02648634, -1.6108699 , -0.32580003, -0.3013221 ],\n",
              "        [-0.3198804 ,  1.3575249 , -0.15776914,  0.5804112 ,\n",
              "          0.4537885 ,  0.9504126 ,  1.6937656 ,  1.2484983 ]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 95
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8p432ETKQGMf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cb89b94c-0ec8-42fa-852e-8b0fffc05bbd"
      },
      "source": [
        "# dense 1: d_mdoel -> d_ff\n",
        "ff_val_1 = tf.keras.layers.Dense(config.d_ff)(sublayer_output)\n",
        "ff_val_1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 32), dtype=float32, numpy=\n",
              "array([[[ 7.7031571e-01, -6.3997334e-01,  9.1023535e-02,  1.3620910e-01,\n",
              "         -6.6182721e-01,  2.5415921e-01,  3.4217364e-01, -3.8794968e-01,\n",
              "         -1.1187077e-02,  3.5169870e-01,  5.4275781e-01, -9.4716631e-02,\n",
              "          4.2744771e-01,  6.4273638e-01, -6.2227696e-03,  4.0499973e-01,\n",
              "          5.0659734e-01, -1.2033993e-01,  8.4220314e-01, -3.2478133e-01,\n",
              "         -1.6106594e-01,  6.0928249e-01, -3.1512687e-01, -2.2889431e-01,\n",
              "          1.0023954e+00,  4.4507617e-01,  2.9803553e-01, -7.0957206e-02,\n",
              "         -1.0344104e+00, -4.9265411e-01,  4.5147824e-01, -2.4699214e-01],\n",
              "        [ 5.7942396e-01, -3.1980455e-02,  7.1110237e-01, -2.4003801e-01,\n",
              "         -8.1442398e-01, -1.2713451e+00, -7.0793545e-01,  4.6208960e-01,\n",
              "         -2.5861040e-01,  1.4218054e+00,  1.8033302e-01, -7.5368106e-02,\n",
              "          5.5270493e-01,  8.0002576e-02, -1.9854778e-01,  1.0054317e+00,\n",
              "          5.7834148e-01, -1.2824627e+00, -1.6690974e-01,  3.5366517e-01,\n",
              "         -1.1411150e+00, -4.9964029e-01,  7.1466655e-01,  7.4253523e-01,\n",
              "          1.1395468e-01, -1.4249539e-01, -8.8010603e-01, -6.5248674e-01,\n",
              "          4.6452349e-01, -2.1270216e-03, -2.0986857e-01,  2.5336069e-01],\n",
              "        [-7.2910178e-01,  5.2506238e-02,  5.7353273e-02, -7.0616293e-01,\n",
              "          5.6857067e-01,  3.1377071e-01,  2.9526538e-01, -3.0134076e-01,\n",
              "         -2.2399893e-01, -7.7470469e-01, -3.5355666e-01,  2.2478200e-01,\n",
              "         -4.9228603e-01, -7.0582581e-01, -3.2548615e-01, -4.7750655e-01,\n",
              "         -3.2047996e-01,  5.7108223e-01, -1.3232665e+00,  4.9953249e-01,\n",
              "         -2.7110964e-01, -1.0204016e+00,  9.8022497e-01,  3.8862637e-01,\n",
              "         -1.2683330e+00,  4.8406664e-02, -3.1995887e-01,  5.7920837e-01,\n",
              "          1.4581698e+00,  5.4629171e-01, -9.5293003e-01,  4.1980624e-01],\n",
              "        [-3.9197278e-01,  8.9776736e-01,  5.9198356e-01, -1.2672752e-01,\n",
              "         -7.3769689e-04, -4.8439696e-01, -4.5251882e-01,  6.7514193e-01,\n",
              "         -8.5150197e-02,  6.5453720e-01, -2.5314039e-01,  8.4378675e-02,\n",
              "          3.8834900e-02, -1.4491270e-01, -5.2905595e-01,  7.3339045e-02,\n",
              "         -5.5365950e-01, -4.3959951e-01, -9.5370579e-01,  2.5929868e-01,\n",
              "         -6.7489493e-01, -7.1621162e-01,  1.3821949e+00,  1.2744250e+00,\n",
              "         -1.3447812e-01, -1.2623402e-01, -3.9556608e-01, -7.9383814e-01,\n",
              "          9.1784841e-01,  2.3702183e-01, -3.6153078e-01,  3.3879828e-01],\n",
              "        [-1.4856537e-01,  3.6119080e-01, -3.1158358e-01,  2.1333542e-01,\n",
              "         -4.9532282e-01,  3.4184954e-01, -2.0787656e-01,  4.5236754e-01,\n",
              "          6.7950231e-01,  2.5161102e-01,  4.3081206e-01, -4.3105596e-01,\n",
              "         -2.6678234e-01,  7.1194607e-01,  2.4308614e-02,  1.9006130e-01,\n",
              "         -2.4239630e-02, -1.7861590e-01,  1.8328184e-01, -1.3510843e-01,\n",
              "          1.4219277e-01,  2.7497920e-01, -3.2282531e-01,  4.8111078e-01,\n",
              "          9.4380188e-01, -2.1392670e-01,  5.0278717e-01, -7.0339793e-01,\n",
              "         -5.8686960e-01, -4.4861898e-02,  1.1911212e-01, -4.1336876e-01],\n",
              "        [ 7.9324722e-02, -9.8293650e-01,  2.0782730e-01, -2.0811744e-01,\n",
              "         -5.1357460e-01, -3.1266052e-01,  4.0670031e-01, -9.3118632e-01,\n",
              "         -6.0891742e-01, -6.8914038e-01,  5.6956542e-01,  4.4738623e-01,\n",
              "         -2.2437228e-01,  2.6204109e-01,  2.5881031e-01,  6.0026282e-01,\n",
              "          1.3433512e+00, -4.9340135e-01,  2.0923750e-01,  2.5450030e-01,\n",
              "         -6.4171642e-02, -1.4561637e-01, -2.8117272e-01, -6.0708076e-01,\n",
              "         -7.1484399e-01, -2.4042147e-01, -5.9817612e-01,  1.3398441e+00,\n",
              "          1.7528468e-01,  2.3007892e-01, -6.2891161e-01,  3.2215783e-01]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 96
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hawd2-NhQQlL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c65a184b-a94d-4f8c-c51b-76d2e36af07f"
      },
      "source": [
        "# relu: 0보다 작은 값은 모두 0\n",
        "ff_val_2 = tf.nn.relu(ff_val_1)\n",
        "ff_val_2"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 32), dtype=float32, numpy=\n",
              "array([[[0.7703157 , 0.        , 0.09102353, 0.1362091 , 0.        ,\n",
              "         0.2541592 , 0.34217364, 0.        , 0.        , 0.3516987 ,\n",
              "         0.5427578 , 0.        , 0.4274477 , 0.6427364 , 0.        ,\n",
              "         0.40499973, 0.50659734, 0.        , 0.84220314, 0.        ,\n",
              "         0.        , 0.6092825 , 0.        , 0.        , 1.0023954 ,\n",
              "         0.44507617, 0.29803553, 0.        , 0.        , 0.        ,\n",
              "         0.45147824, 0.        ],\n",
              "        [0.57942396, 0.        , 0.71110237, 0.        , 0.        ,\n",
              "         0.        , 0.        , 0.4620896 , 0.        , 1.4218054 ,\n",
              "         0.18033302, 0.        , 0.55270493, 0.08000258, 0.        ,\n",
              "         1.0054317 , 0.5783415 , 0.        , 0.        , 0.35366517,\n",
              "         0.        , 0.        , 0.71466655, 0.74253523, 0.11395468,\n",
              "         0.        , 0.        , 0.        , 0.4645235 , 0.        ,\n",
              "         0.        , 0.2533607 ],\n",
              "        [0.        , 0.05250624, 0.05735327, 0.        , 0.5685707 ,\n",
              "         0.3137707 , 0.29526538, 0.        , 0.        , 0.        ,\n",
              "         0.        , 0.224782  , 0.        , 0.        , 0.        ,\n",
              "         0.        , 0.        , 0.57108223, 0.        , 0.4995325 ,\n",
              "         0.        , 0.        , 0.98022497, 0.38862637, 0.        ,\n",
              "         0.04840666, 0.        , 0.5792084 , 1.4581698 , 0.5462917 ,\n",
              "         0.        , 0.41980624],\n",
              "        [0.        , 0.89776736, 0.59198356, 0.        , 0.        ,\n",
              "         0.        , 0.        , 0.67514193, 0.        , 0.6545372 ,\n",
              "         0.        , 0.08437867, 0.0388349 , 0.        , 0.        ,\n",
              "         0.07333905, 0.        , 0.        , 0.        , 0.25929868,\n",
              "         0.        , 0.        , 1.3821949 , 1.274425  , 0.        ,\n",
              "         0.        , 0.        , 0.        , 0.9178484 , 0.23702183,\n",
              "         0.        , 0.33879828],\n",
              "        [0.        , 0.3611908 , 0.        , 0.21333542, 0.        ,\n",
              "         0.34184954, 0.        , 0.45236754, 0.6795023 , 0.25161102,\n",
              "         0.43081206, 0.        , 0.        , 0.71194607, 0.02430861,\n",
              "         0.1900613 , 0.        , 0.        , 0.18328184, 0.        ,\n",
              "         0.14219277, 0.2749792 , 0.        , 0.48111078, 0.9438019 ,\n",
              "         0.        , 0.5027872 , 0.        , 0.        , 0.        ,\n",
              "         0.11911212, 0.        ],\n",
              "        [0.07932472, 0.        , 0.2078273 , 0.        , 0.        ,\n",
              "         0.        , 0.4067003 , 0.        , 0.        , 0.        ,\n",
              "         0.5695654 , 0.44738623, 0.        , 0.2620411 , 0.2588103 ,\n",
              "         0.6002628 , 1.3433512 , 0.        , 0.2092375 , 0.2545003 ,\n",
              "         0.        , 0.        , 0.        , 0.        , 0.        ,\n",
              "         0.        , 0.        , 1.3398441 , 0.17528468, 0.23007892,\n",
              "         0.        , 0.32215783]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 97
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W20iKxKGQYly",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ffb58889-806d-4401-c4b4-853293befa5d"
      },
      "source": [
        "# dense 2: d_ff -> d_model\n",
        "ff_val = tf.keras.layers.Dense(config.d_model)(ff_val_2)\n",
        "ff_val"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 8), dtype=float32, numpy=\n",
              "array([[[ 0.05343075, -0.12957534, -0.12799206,  0.22800428,\n",
              "          0.3755629 ,  0.02002761, -0.52819777,  0.3447255 ],\n",
              "        [ 0.43111062,  0.47392774,  0.10379801,  0.6578367 ,\n",
              "         -0.13527879,  0.09563409, -0.27491823, -0.70665586],\n",
              "        [ 0.59465665,  0.9119929 , -0.3263978 ,  0.37083977,\n",
              "          0.6611173 , -0.00635345,  0.1908662 , -0.04102041],\n",
              "        [ 0.39177606,  0.548897  , -0.1065183 ,  0.2984062 ,\n",
              "         -0.07401261,  0.02801819,  0.4292276 , -0.32901528],\n",
              "        [ 0.16526428,  0.12299663,  0.24812302,  0.7795001 ,\n",
              "          0.2166494 ,  0.48855445,  0.11982807,  0.90903956],\n",
              "        [-0.5053141 , -0.1995892 , -0.571194  ,  0.34657496,\n",
              "          0.03725423, -0.57696253, -1.0532656 ,  0.19136594]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 98
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xDuEmCeoN2lq",
        "colab_type": "text"
      },
      "source": [
        "### Class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9nUJHhLfQhdE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class PositionWiseFeedForward(tf.keras.layers.Layer):\n",
        "    \"\"\"\n",
        "    Position Wise Feed Forward Class\n",
        "    \"\"\"\n",
        "    def __init__(self, config, name=\"feed_forward\"):\n",
        "        \"\"\"\n",
        "        생성자\n",
        "        :param config: Config 객체\n",
        "        :param name: layer name\n",
        "        \"\"\"\n",
        "        super().__init__(name=name)\n",
        "\n",
        "        self.W_1 = tf.keras.layers.Dense(config.d_ff, activation=tf.nn.relu)\n",
        "        self.W_2 = tf.keras.layers.Dense(config.d_model)\n",
        "\n",
        "    def call(self, inputs):\n",
        "        \"\"\"\n",
        "        layer 실행\n",
        "        :param inputs: inputs\n",
        "        :return ff_val: feed forward 실행 결과\n",
        "        \"\"\"\n",
        "        ff_val = self.W_2(self.W_1(inputs))\n",
        "        return ff_val"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5rTITKdIN5nU",
        "colab_type": "text"
      },
      "source": [
        "### Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2zBze7AoQp-_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7e3ca3c4-d702-4ce7-a8e7-9cf53eb65aca"
      },
      "source": [
        "# feed-forward class 동작 확인\n",
        "feed_forward = PositionWiseFeedForward(config)\n",
        "\n",
        "ff_val = feed_forward(sublayer_output)\n",
        "# 입력과 출력의 shape가 같의면 정상 (W_1, W_2 값이 랜덤초기화 되므로 값은 변경 됨)\n",
        "assert ff_val.shape == sublayer_output.shape\n",
        "ff_val"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 8), dtype=float32, numpy=\n",
              "array([[[ 0.3864193 ,  0.6548004 , -0.9338118 ,  0.26108032,\n",
              "          0.03351755,  1.0447208 ,  0.7750368 , -0.12114957],\n",
              "        [-0.59722066,  0.18925115, -0.06597053, -0.37621725,\n",
              "         -0.23455417,  1.5406342 ,  0.4521676 ,  0.73663914],\n",
              "        [-0.04632387,  0.10500703,  0.22665873, -0.3026586 ,\n",
              "         -0.7785901 ,  0.04514271,  0.0512608 , -0.0021835 ],\n",
              "        [-0.77620435, -0.29549295,  0.35187596, -0.13449736,\n",
              "         -0.13364094,  0.82650733, -0.20703031,  0.51752913],\n",
              "        [ 0.26921114,  0.25551227, -0.37782532,  0.05918773,\n",
              "         -0.3883301 ,  0.08096398,  0.00456275,  0.4402079 ],\n",
              "        [ 0.6045598 ,  0.11301185, -0.6297971 ,  0.12454662,\n",
              "         -0.6757138 ,  0.29747704,  0.1991685 , -0.18869305]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 100
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oCkXJQIUlETb",
        "colab_type": "text"
      },
      "source": [
        "## Encoder Layer\n",
        "\n",
        "![](https://paul-hyun.github.io/assets/transformer/encoder_layer.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eZXtO5pKONH1",
        "colab_type": "text"
      },
      "source": [
        "### Tutorial"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wQ0BTLQmOXEE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "76a4b792-39d0-41de-bd86-1eb40f6f11fa"
      },
      "source": [
        "# encoder self attetnion mask\n",
        "self_mask = get_pad_mask(enc_tokens)\n",
        "self_mask"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 1, 6), dtype=float32, numpy=array([[[0., 0., 0., 0., 0., 1.]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 101
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M3V8lHcKOSYN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "50914a65-ccec-4efb-b00a-3ade04a0f2c6"
      },
      "source": [
        "# encoder embedding\n",
        "enc_embed = embedding(enc_tokens)\n",
        "enc_embed"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 8), dtype=float32, numpy=\n",
              "array([[[ 0.19626832,  1.6160245 , -1.5125287 ,  0.8802097 ,\n",
              "          0.43964735, -0.49681696,  0.9233131 , -1.0172113 ],\n",
              "        [-1.851637  , -0.52205473, -1.1127012 ,  0.556745  ,\n",
              "         -1.2216276 ,  0.70823485, -0.7989583 , -0.48866653],\n",
              "        [-0.39263427, -1.7194668 ,  1.8227654 , -0.15567164,\n",
              "         -0.6654087 ,  0.68538827,  0.68061614,  0.74703187],\n",
              "        [-1.3610662 , -1.8012775 , -0.09980725, -0.69201565,\n",
              "         -1.6705056 , -0.07607771, -0.5028183 , -0.22716261],\n",
              "        [-0.20390666,  0.557556  , -0.40127853,  0.23037532,\n",
              "          0.02648634, -1.6108699 , -0.32580003, -0.3013221 ],\n",
              "        [-0.3198804 ,  1.3575249 , -0.15776914,  0.5804112 ,\n",
              "          0.4537885 ,  0.9504126 ,  1.6937656 ,  1.2484983 ]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 102
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9siOPgyXOlmN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "84569dfa-1ab6-48b1-e7cb-776f6207ed4e"
      },
      "source": [
        "# MultiHeadAttention\n",
        "self_attention = MultiHeadAttention(config)\n",
        "self_attn_val = self_attention(enc_embed, enc_embed, enc_embed, self_mask)\n",
        "self_attn_val"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 8), dtype=float32, numpy=\n",
              "array([[[ 0.06125213, -0.7417859 , -0.24070373, -0.13314433,\n",
              "          0.7362003 ,  0.05651649, -0.24332067, -0.39187843],\n",
              "        [-0.18421775, -0.63064814, -0.3743923 , -0.06857643,\n",
              "          0.58380187,  0.17402935, -0.20345758, -0.7997295 ],\n",
              "        [-0.09924531,  0.4825187 ,  0.6180561 , -0.7602583 ,\n",
              "          0.32308692, -0.52801055, -0.5650271 ,  0.2589812 ],\n",
              "        [-0.34235   , -0.10044113,  0.05761166, -0.13948211,\n",
              "          0.07773675,  0.08463562, -0.31411588, -0.46418273],\n",
              "        [ 0.12851135, -0.60527325, -0.07221869, -0.29906297,\n",
              "          0.7786446 , -0.08854971, -0.33206272, -0.179259  ],\n",
              "        [ 0.21461186, -0.42240644, -0.06978095, -0.3544655 ,\n",
              "          0.826947  , -0.15981814, -0.30214188, -0.10174474]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 103
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hGI5NdiaO7FS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "57971ace-35f2-48cd-9a54-165176898d4c"
      },
      "source": [
        "norm1 = tf.keras.layers.LayerNormalization(epsilon=config.layernorm_epsilon)\n",
        "dropout = tf.keras.layers.Dropout(config.dropout)\n",
        "norm1_val = norm1(enc_embed + dropout(self_attn_val))\n",
        "norm1_val"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 8), dtype=float32, numpy=\n",
              "array([[[ 0.23391862,  0.8324778 , -1.7176285 ,  0.70904917,\n",
              "          1.1252061 , -0.44335526,  0.6439511 , -1.3836191 ],\n",
              "        [-1.3507801 , -0.40145454, -0.7609007 ,  1.3623655 ,\n",
              "          0.15200171,  1.7859902 , -0.23990679, -0.5473152 ],\n",
              "        [-0.53051513, -1.2079626 ,  2.1360192 , -0.91607946,\n",
              "         -0.3945309 ,  0.05981676,  0.02182073,  0.83143145],\n",
              "        [-1.1077341 , -1.3979136 ,  1.3231605 ,  0.1681602 ,\n",
              "         -0.9458218 ,  1.3974291 ,  0.18947141,  0.37324786],\n",
              "        [ 0.39393005,  0.43556786, -0.20495962,  0.4040208 ,\n",
              "          1.7185609 , -2.0491915 , -0.4823123 , -0.21561636],\n",
              "        [-1.3229213 ,  0.43036264, -1.5289929 , -0.7647515 ,\n",
              "          1.0128044 ,  0.18680753,  1.1996758 ,  0.78701496]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 104
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eFpRZ-R3PV2C",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "301c1dc2-c04b-4cb1-b379-6145765fe93e"
      },
      "source": [
        "ffn = PositionWiseFeedForward(config)\n",
        "ffn_val = ffn(norm1_val)\n",
        "ffn_val"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 8), dtype=float32, numpy=\n",
              "array([[[-0.3325595 ,  0.6135318 ,  0.20051412, -0.07957099,\n",
              "          0.7559771 , -0.07830811,  0.7206068 , -0.29344258],\n",
              "        [-0.41146573,  0.05666243, -0.22377066, -0.3695033 ,\n",
              "          0.26574633, -0.26741296,  0.5624498 , -0.14689448],\n",
              "        [ 0.5862982 , -0.6090269 , -0.23588449,  0.7762842 ,\n",
              "          0.48423082, -0.14400674,  1.137095  , -0.17276293],\n",
              "        [ 0.06423843, -0.7943398 ,  0.01329803,  0.21145219,\n",
              "          0.561966  ,  0.07592963,  0.5929384 ,  0.30395535],\n",
              "        [ 0.09608704,  1.0420611 , -0.5916425 ,  0.06332767,\n",
              "          0.7061918 , -0.1245482 ,  1.2926265 , -0.30812192],\n",
              "        [ 0.4622246 , -0.02264547, -0.22483435, -0.15210834,\n",
              "         -0.618608  , -0.45302588,  0.32223728, -0.51863307]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 105
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M-nhPgYpPft4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ab56165f-0927-43d8-b3bd-255946f3dbe9"
      },
      "source": [
        "norm2 = tf.keras.layers.LayerNormalization(epsilon=config.layernorm_epsilon)\n",
        "enc_out = norm2(norm1_val + dropout(ffn_val))\n",
        "enc_out"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 8), dtype=float32, numpy=\n",
              "array([[[-0.22505724,  0.98627955, -1.3374442 ,  0.345944  ,\n",
              "          1.3275487 , -0.55679744,  0.922404  , -1.4628772 ],\n",
              "        [-1.6727693 , -0.2742958 , -0.90560675,  1.0454468 ,\n",
              "          0.47803363,  1.5641223 ,  0.38410345, -0.6190345 ],\n",
              "        [-0.16865917, -2.005106  ,  1.6399181 , -0.36044392,\n",
              "         -0.1354002 , -0.3059172 ,  0.9130763 ,  0.42253223],\n",
              "        [-0.99559164, -1.9712933 ,  1.0258316 ,  0.2131306 ,\n",
              "         -0.435324  ,  1.1421082 ,  0.55524814,  0.46589047],\n",
              "        [ 0.1634807 ,  0.9040362 , -0.80128396,  0.14648274,\n",
              "          1.6142321 , -1.8339232 ,  0.40365377, -0.59667844],\n",
              "        [-0.7451013 ,  0.5859788 , -1.6823572 , -0.80403924,\n",
              "          0.57179004, -0.12125265,  1.7552218 ,  0.43975964]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 106
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F8_N3eEDOA2z",
        "colab_type": "text"
      },
      "source": [
        "### Class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z3PGFf56lnVa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class EncoderLayer(tf.keras.layers.Layer):\n",
        "    \"\"\"\n",
        "    Encoder Layer Class\n",
        "    \"\"\"\n",
        "    def __init__(self, config, name=\"encoder_layer\"):\n",
        "        \"\"\"\n",
        "        생성자\n",
        "        :param config: Config 객체\n",
        "        :param name: layer name\n",
        "        \"\"\"\n",
        "        super().__init__(name=name)\n",
        "\n",
        "        self.self_attention = MultiHeadAttention(config)\n",
        "        self.norm1 = tf.keras.layers.LayerNormalization(epsilon=config.layernorm_epsilon)\n",
        "\n",
        "        self.ffn = PositionWiseFeedForward(config)\n",
        "        self.norm2 = tf.keras.layers.LayerNormalization(epsilon=config.layernorm_epsilon)\n",
        "\n",
        "        self.dropout = tf.keras.layers.Dropout(config.dropout)\n",
        " \n",
        "    def call(self, enc_embed, self_mask):\n",
        "        \"\"\"\n",
        "        layer 실행\n",
        "        :param enc_embed: enc_embed 또는 이전 EncoderLayer의 출력\n",
        "        :param self_mask: enc_tokens의 pad mask\n",
        "        :return enc_out: EncoderLayer 실행 결과\n",
        "        \"\"\"\n",
        "        self_attn_val = self.self_attention(enc_embed, enc_embed, enc_embed, self_mask)\n",
        "        norm1_val = self.norm1(enc_embed + self.dropout(self_attn_val))\n",
        "\n",
        "        ffn_val = self.ffn(norm1_val)\n",
        "        enc_out = self.norm2(norm1_val + self.dropout(ffn_val))\n",
        "\n",
        "        return enc_out"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I9zNYIgzPwW3",
        "colab_type": "text"
      },
      "source": [
        "### Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K3rfiHQ6nJt6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c33bf0fe-3387-41ab-ea8b-069bdf4f5d9d"
      },
      "source": [
        "# EncoderLayer 기능 확인: 결과의 shape이 enc_embed과 같으면 성공\n",
        "encoder_layer = EncoderLayer(config)\n",
        "enc_out = encoder_layer(enc_embed, self_mask)\n",
        "assert enc_embed.shape == enc_out.shape\n",
        "enc_out"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 8), dtype=float32, numpy=\n",
              "array([[[ 1.3676884 ,  1.1106462 , -0.47449332,  0.5410458 ,\n",
              "         -0.4758251 , -1.8929135 ,  0.43532416, -0.6114727 ],\n",
              "        [-0.9921528 ,  0.11668632, -0.9610562 ,  1.8877946 ,\n",
              "         -1.0831547 ,  1.1038936 , -0.27301744,  0.20100632],\n",
              "        [-0.45617127, -2.2258744 ,  0.2684582 ,  0.9690313 ,\n",
              "         -0.6119591 ,  0.8086656 ,  0.5392151 ,  0.70863485],\n",
              "        [-0.5972522 , -1.7844365 , -0.00578466,  1.4824632 ,\n",
              "         -0.85839444,  1.1552196 ,  0.29377365,  0.31441143],\n",
              "        [ 1.0394422 ,  0.29920736,  0.12604168,  0.2683921 ,\n",
              "          0.4059655 , -2.5473633 ,  0.18588047,  0.22243404],\n",
              "        [-1.193143  , -0.7259167 , -0.794822  , -0.01026419,\n",
              "         -0.18535131,  0.03583572,  0.6497321 ,  2.2239294 ]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 108
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "teJEuUHAlI5f",
        "colab_type": "text"
      },
      "source": [
        "## Decoder Layer\n",
        "\n",
        "![](https://paul-hyun.github.io/assets/transformer/decoder_layer.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L9be5hBhR0k9",
        "colab_type": "text"
      },
      "source": [
        "### Tutorial"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MUVYeyNwQuTV",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9fc940e0-4208-4892-ac32-64f7bf1d01cc"
      },
      "source": [
        "# decoder self attention ahead mask\n",
        "self_mask = get_ahead_mask(dec_tokens)\n",
        "self_mask"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 5), dtype=float32, numpy=\n",
              "array([[[0., 1., 1., 1., 1.],\n",
              "        [0., 0., 1., 1., 1.],\n",
              "        [0., 0., 0., 1., 1.],\n",
              "        [0., 0., 0., 0., 1.],\n",
              "        [0., 0., 0., 0., 1.]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 109
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5qjjwRI2Q3j-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ad543185-248d-4262-940e-57dc14065f50"
      },
      "source": [
        "# encoder & decoder attention mask\n",
        "ende_mask = get_pad_mask(enc_tokens)\n",
        "ende_mask"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 1, 6), dtype=float32, numpy=array([[[0., 0., 0., 0., 0., 1.]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 110
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M3XHbm2lQWFc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e09dd83c-f7f9-4fa0-f3a8-d4519f14955d"
      },
      "source": [
        "enc_out = embedding(enc_tokens)\n",
        "enc_out"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 8), dtype=float32, numpy=\n",
              "array([[[ 0.19626832,  1.6160245 , -1.5125287 ,  0.8802097 ,\n",
              "          0.43964735, -0.49681696,  0.9233131 , -1.0172113 ],\n",
              "        [-1.851637  , -0.52205473, -1.1127012 ,  0.556745  ,\n",
              "         -1.2216276 ,  0.70823485, -0.7989583 , -0.48866653],\n",
              "        [-0.39263427, -1.7194668 ,  1.8227654 , -0.15567164,\n",
              "         -0.6654087 ,  0.68538827,  0.68061614,  0.74703187],\n",
              "        [-1.3610662 , -1.8012775 , -0.09980725, -0.69201565,\n",
              "         -1.6705056 , -0.07607771, -0.5028183 , -0.22716261],\n",
              "        [-0.20390666,  0.557556  , -0.40127853,  0.23037532,\n",
              "          0.02648634, -1.6108699 , -0.32580003, -0.3013221 ],\n",
              "        [-0.3198804 ,  1.3575249 , -0.15776914,  0.5804112 ,\n",
              "          0.4537885 ,  0.9504126 ,  1.6937656 ,  1.2484983 ]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 111
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uYHuqpojQckk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e9bd30d5-f9bc-47b4-bc45-fcd8ff60a023"
      },
      "source": [
        "dec_embed = embedding(dec_tokens)\n",
        "dec_embed"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 8), dtype=float32, numpy=\n",
              "array([[[-1.8684977 , -0.10887106, -1.6283252 , -0.09412991,\n",
              "         -0.9135378 , -0.5230187 ,  0.18054976,  1.0659311 ],\n",
              "        [-1.851637  , -0.52205473, -1.1127012 ,  0.556745  ,\n",
              "         -1.2216276 ,  0.70823485, -0.7989583 , -0.48866653],\n",
              "        [-0.20390666,  0.557556  , -0.40127853,  0.23037532,\n",
              "          0.02648634, -1.6108699 , -0.32580003, -0.3013221 ],\n",
              "        [ 0.8284642 ,  1.1036007 , -0.78627795,  1.6918567 ,\n",
              "         -0.34767905,  1.3073093 ,  0.2444622 , -0.7029787 ],\n",
              "        [-0.3198804 ,  1.3575249 , -0.15776914,  0.5804112 ,\n",
              "          0.4537885 ,  0.9504126 ,  1.6937656 ,  1.2484983 ]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 112
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R6YNM4zPQpn9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1eaba855-c110-47f5-ee3f-6571d811923a"
      },
      "source": [
        "# MultiHeadAttention\n",
        "self_attention = MultiHeadAttention(config)\n",
        "self_attn_val = self_attention(dec_embed, dec_embed, dec_embed, self_mask)\n",
        "self_attn_val"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 8), dtype=float32, numpy=\n",
              "array([[[-0.2852324 ,  0.4208145 , -0.70683515,  1.2605366 ,\n",
              "          0.80964255, -0.70027095, -0.20946491, -0.98482513],\n",
              "        [-0.39821142,  0.49257648, -0.88418823,  0.88798165,\n",
              "          0.6358985 , -0.43642837, -0.03065154, -0.6418692 ],\n",
              "        [-0.3649325 ,  0.6137138 , -0.6272023 ,  0.31882334,\n",
              "          0.10736643, -0.12597029, -0.40170333,  0.11896834],\n",
              "        [ 0.07283233, -0.05366343, -0.6364631 ,  0.10185602,\n",
              "          0.01400785,  0.04241411, -0.00430873,  0.11441461],\n",
              "        [ 0.0622538 ,  0.08478439, -0.58896554,  0.46568584,\n",
              "          0.27507114, -0.19285972, -0.00310168, -0.3683273 ]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 113
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b9mNNIyURIVz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "01d7c846-2c22-44d7-c9dd-6f98ec4fee77"
      },
      "source": [
        "norm1 = tf.keras.layers.LayerNormalization(epsilon=config.layernorm_epsilon)\n",
        "dropout = tf.keras.layers.Dropout(config.dropout)\n",
        "norm1_val = norm1(dec_embed + dropout(self_attn_val))\n",
        "norm1_val"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 8), dtype=float32, numpy=\n",
              "array([[[-1.3941187 ,  0.73033136, -1.5504408 ,  1.4665457 ,\n",
              "          0.3720404 , -0.59244156,  0.43664402,  0.53143936],\n",
              "        [-1.4223993 ,  0.53723574, -1.199145  ,  1.8383268 ,\n",
              "          0.04630465,  0.803141  , -0.1689375 , -0.43452603],\n",
              "        [-0.31332773,  1.7050711 , -0.8464782 ,  0.98351413,\n",
              "          0.5017434 , -1.6681231 , -0.49736652,  0.13496701],\n",
              "        [ 0.51534015,  0.6605363 , -1.7548413 ,  1.3870759 ,\n",
              "         -0.6910091 ,  0.9533753 , -0.13048194, -0.93999517],\n",
              "        [-1.2389445 ,  0.9773583 , -1.8766224 ,  0.46079394,\n",
              "          0.04719358,  0.08460253,  1.3011523 ,  0.24446683]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 114
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cRZ-kXIeREfI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "353f548c-bb9e-415d-c54f-7eb7b96dfd35"
      },
      "source": [
        "# MultiHeadAttention\n",
        "ende_attn = MultiHeadAttention(config)\n",
        "ende_attn_val = ende_attn(norm1_val, enc_out, enc_out, ende_mask)\n",
        "ende_attn_val"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 8), dtype=float32, numpy=\n",
              "array([[[ 0.1556097 , -0.47930825, -0.10339537,  0.58925897,\n",
              "         -0.55039364,  0.41632497, -0.36272573, -0.715899  ],\n",
              "        [-0.08362544, -0.66093415, -0.8670446 ,  0.20810007,\n",
              "         -0.353948  ,  0.9197676 , -0.26825485, -0.30807742],\n",
              "        [ 0.40666932, -0.40180013,  0.29215094,  0.66815907,\n",
              "         -0.47366628,  0.16566025, -0.4442336 , -0.8687927 ],\n",
              "        [-0.08692276, -0.5690259 , -0.9159167 ,  0.15064606,\n",
              "         -0.26811436,  0.8909161 , -0.24519931, -0.15989664],\n",
              "        [-0.02124169, -0.61071146, -0.74210215,  0.38190126,\n",
              "         -0.54190004,  0.85552585, -0.3627075 , -0.45385766]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 115
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eSXb8l89Ratj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "26fc9b1a-dc9e-4648-b8e7-ade7e5eca09e"
      },
      "source": [
        "norm2 = tf.keras.layers.LayerNormalization(epsilon=config.layernorm_epsilon)\n",
        "norm2_val = norm2(norm1_val + dropout(ende_attn_val))\n",
        "norm2_val"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 8), dtype=float32, numpy=\n",
              "array([[[-1.0721947 ,  0.3702535 , -1.4743935 ,  2.1179862 ,\n",
              "         -0.04555035, -0.04338439,  0.19874692, -0.0514637 ],\n",
              "        [-0.9940957 ,  0.03967612, -1.4130145 ,  1.6626028 ,\n",
              "         -0.09788699,  1.42066   , -0.19477011, -0.42317158],\n",
              "        [ 0.1713014 ,  1.3534757 , -0.46151027,  1.6938858 ,\n",
              "          0.10753404, -1.3878962 , -0.8398993 , -0.6368911 ],\n",
              "        [ 0.4201039 ,  0.1755944 , -1.8291161 ,  1.2251794 ,\n",
              "         -0.58690095,  1.4476715 , -0.16346896, -0.6890632 ],\n",
              "        [-0.9123304 ,  0.47051668, -2.0671206 ,  0.8751692 ,\n",
              "         -0.2616543 ,  0.95798975,  0.9565587 , -0.01912916]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 116
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8oG4_TLaRnbh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "88b3723e-ca2c-4c89-9eaf-d0ff46f6a0d9"
      },
      "source": [
        "ffn = PositionWiseFeedForward(config)\n",
        "ffn_val = ffn(norm2_val)\n",
        "ffn_val"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 8), dtype=float32, numpy=\n",
              "array([[[ 0.06123817,  0.31636155,  1.4304272 ,  0.57025176,\n",
              "         -0.07129043, -0.2288833 , -0.58413416,  0.4339293 ],\n",
              "        [ 0.03198646,  0.07162552,  0.78945005,  0.7710272 ,\n",
              "         -0.128409  , -0.33248097,  0.03863737,  0.6759969 ],\n",
              "        [-0.834555  ,  0.6630019 ,  1.5049067 ,  1.1795566 ,\n",
              "          0.14338756,  0.25348714, -1.003786  ,  0.18767136],\n",
              "        [-0.33797997,  0.44553936,  0.5964004 ,  0.91889906,\n",
              "         -0.19107962, -0.34810108, -0.05615854,  0.90421027],\n",
              "        [ 0.42497897,  0.44764152,  1.067889  ,  0.8069382 ,\n",
              "         -0.21009508, -0.49760538, -0.04377609,  0.79625756]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 117
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7ixzZBNURq_1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "066c883f-fa30-4970-8930-a47dafc216d2"
      },
      "source": [
        "norm3 = tf.keras.layers.LayerNormalization(epsilon=config.layernorm_epsilon)\n",
        "dec_out = norm2(norm2_val + dropout(ffn_val))\n",
        "dec_out"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 8), dtype=float32, numpy=\n",
              "array([[[-1.2041254 ,  0.42860657, -0.27406985,  2.3537765 ,\n",
              "         -0.34416085, -0.49365118, -0.6024501 ,  0.13607427],\n",
              "        [-1.1928285 , -0.12746477, -0.8568218 ,  2.1774535 ,\n",
              "         -0.46253148,  0.8420893 , -0.39289424,  0.01299795],\n",
              "        [-0.6121968 ,  1.1614136 ,  0.51736873,  1.7286054 ,\n",
              "         -0.00713963, -0.9240364 , -1.3934789 , -0.4705361 ],\n",
              "        [-0.1597462 ,  0.38062987, -1.4779186 ,  1.9074349 ,\n",
              "         -1.0220308 ,  0.8602793 , -0.4622625 , -0.02638588],\n",
              "        [-0.98113656,  0.66763175, -1.5816104 ,  1.5638016 ,\n",
              "         -0.9628342 ,  0.1306287 ,  0.6613258 ,  0.5021932 ]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 118
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XpS8LxinR4HZ",
        "colab_type": "text"
      },
      "source": [
        "### Class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iuaIfX6wolCJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class DecoderLayer(tf.keras.layers.Layer):\n",
        "    \"\"\"\n",
        "    Decoder Layer Class\n",
        "    \"\"\"\n",
        "    def __init__(self, config, name=\"decoder_layer\"):\n",
        "        \"\"\"\n",
        "        생성자\n",
        "        :param config: Config 객체\n",
        "        :param name: layer name\n",
        "        \"\"\"\n",
        "        super().__init__(name=name)\n",
        "\n",
        "        self.self_attention = MultiHeadAttention(config)\n",
        "        self.norm1 = tf.keras.layers.LayerNormalization(epsilon=config.layernorm_epsilon)\n",
        "\n",
        "        self.ende_attn = MultiHeadAttention(config)\n",
        "        self.norm2 = tf.keras.layers.LayerNormalization(epsilon=config.layernorm_epsilon)\n",
        "\n",
        "        self.ffn = PositionWiseFeedForward(config)\n",
        "        self.norm3 = tf.keras.layers.LayerNormalization(epsilon=config.layernorm_epsilon)\n",
        "\n",
        "        self.dropout = tf.keras.layers.Dropout(config.dropout)\n",
        "\n",
        "    def call(self, dec_embed, enc_out, self_mask, ende_mask):\n",
        "        \"\"\"\n",
        "        layer 실행\n",
        "        :param dec_embed: dec_embed 또는 이전 DecoderLayer의 출력\n",
        "        :param enc_out: 마지막 EncoderLayer의 출력\n",
        "        :param self_mask: dec_tokens의 ahead mask\n",
        "        :param ende_mask: enc_tokens의 pad mask\n",
        "        :return dec_out: DecoderLayer 실행 결과\n",
        "        \"\"\"\n",
        "        self_attn_val = self.self_attention(dec_embed, dec_embed, dec_embed, self_mask)\n",
        "        norm1_val = self.norm1(dec_embed + self.dropout(self_attn_val))\n",
        "\n",
        "        ende_attn_val = self.ende_attn(norm1_val, enc_out, enc_out, ende_mask)\n",
        "        norm2_val = self.norm2(norm1_val + self.dropout(ende_attn_val))\n",
        "\n",
        "        ffn_val = self.ffn(norm2_val)\n",
        "        dec_out = self.norm3(norm2_val + self.dropout(ffn_val))\n",
        "\n",
        "        return dec_out"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hsohSU2sR7QF",
        "colab_type": "text"
      },
      "source": [
        "### Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "rXgIQENZp1Je",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "39aa9e76-45a9-4b63-be7f-c55ba26a2508"
      },
      "source": [
        "# Decoder 실행: 결과의 shape이 dec_embed 같으면 성공(?)\n",
        "decoder_layer = DecoderLayer(config)\n",
        "dec_out = decoder_layer(dec_embed, enc_out, self_mask, ende_mask)\n",
        "assert dec_embed.shape == dec_out.shape\n",
        "dec_out"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 8), dtype=float32, numpy=\n",
              "array([[[-1.0030594 ,  0.580575  , -1.697617  , -0.36998567,\n",
              "         -0.06975577,  0.05840588,  0.75268304,  1.7487538 ],\n",
              "        [-1.1175425 , -0.8313827 , -0.7349679 , -0.7224272 ,\n",
              "          0.15881291,  0.83438903,  0.38462862,  2.0284898 ],\n",
              "        [-1.3698311 ,  1.086454  , -1.0407254 ,  0.42365548,\n",
              "          0.6603588 , -1.1805588 ,  1.3566747 ,  0.06397215],\n",
              "        [-1.1084545 ,  1.1683538 , -0.28642395,  1.0768398 ,\n",
              "         -0.19314095,  0.10367056,  1.0011281 , -1.7619731 ],\n",
              "        [-1.9044942 ,  0.906461  , -1.0239764 , -0.09501362,\n",
              "          0.37727764, -0.11386959,  1.4811611 ,  0.37245405]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 120
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fC79J4CYuWF7",
        "colab_type": "text"
      },
      "source": [
        "## Transformer\n",
        "\n",
        "![](https://paul-hyun.github.io/assets/transformer/transformer.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MRwW6_IRTtWv",
        "colab_type": "text"
      },
      "source": [
        "### Tutorial"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E5-0K2HaT4S8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "62621a5c-05a4-4792-95c7-f4b3f29d90ba"
      },
      "source": [
        "# encoder self attention mask\n",
        "enc_self_mask = get_pad_mask(enc_tokens)\n",
        "enc_self_mask"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 1, 6), dtype=float32, numpy=array([[[0., 0., 0., 0., 0., 1.]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 121
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sYbavBWVULDW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "393a31d2-7f40-4201-a72c-388fd52f4cda"
      },
      "source": [
        "# decoder self attention mask\n",
        "dec_self_mask = get_ahead_mask(dec_tokens)\n",
        "dec_self_mask"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 5), dtype=float32, numpy=\n",
              "array([[[0., 1., 1., 1., 1.],\n",
              "        [0., 0., 1., 1., 1.],\n",
              "        [0., 0., 0., 1., 1.],\n",
              "        [0., 0., 0., 0., 1.],\n",
              "        [0., 0., 0., 0., 1.]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 122
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jsCPrXSHUXyW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4208408c-1e3c-4817-9069-00c2693581e3"
      },
      "source": [
        "# encoder decoder attention mask\n",
        "ende_attn_mask = get_pad_mask(enc_tokens)\n",
        "ende_attn_mask"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 1, 6), dtype=float32, numpy=array([[[0., 0., 0., 0., 0., 1.]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 123
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o4TFj2QvTv9x",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# embedding 선언\n",
        "embedding = SharedEmbedding(config)\n",
        "position = PositionalEmbedding(config)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CDejHFtYUgJS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "60a9e7c7-0df6-41d9-8d7f-17e7cc2770dd"
      },
      "source": [
        "# encoder embedding\n",
        "enc_embed = embedding(enc_tokens) + position(enc_tokens)\n",
        "enc_embed"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 8), dtype=float32, numpy=\n",
              "array([[[ 0.08233406,  2.8566623 , -1.413196  ,  1.9292356 ,\n",
              "          1.0852543 ,  2.3589113 ,  1.0987881 , -0.57102036],\n",
              "        [ 1.3721294 ,  0.86170125,  1.0635568 ,  2.274029  ,\n",
              "         -0.4628705 , -0.1085456 , -0.69250786,  0.23946345],\n",
              "        [-0.18527573,  0.14757022,  0.17361866,  1.924506  ,\n",
              "         -1.076756  , -0.657716  ,  0.59556913, -0.09281492],\n",
              "        [ 2.06635   , -1.9722767 ,  0.16101182,  1.0998178 ,\n",
              "          0.32978362,  0.16121948, -0.7051664 ,  0.13986266],\n",
              "        [ 0.9735252 , -1.039552  ,  0.6461056 ,  2.1154597 ,\n",
              "         -0.73406786,  0.39392954, -1.1797982 ,  1.0604619 ],\n",
              "        [-1.0512958 ,  0.5968548 , -0.70903325,  0.7909003 ,\n",
              "          1.8263443 ,  1.3972509 , -0.44632512, -0.08020413]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 125
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PU-JPl-lUp0b",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5c83bbd1-216e-42a7-ffed-d8d7a7e1774d"
      },
      "source": [
        "# encoder embedding\n",
        "dec_embed = embedding(dec_tokens) + position(dec_tokens)\n",
        "dec_embed"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 8), dtype=float32, numpy=\n",
              "array([[[-0.15543607,  1.4198871 ,  0.8532211 ,  0.48257583,\n",
              "         -0.01898028,  0.6980718 , -0.94074667,  1.5109348 ],\n",
              "        [ 1.3721294 ,  0.86170125,  1.0635568 ,  2.274029  ,\n",
              "         -0.4628705 , -0.1085456 , -0.69250786,  0.23946345],\n",
              "        [ 2.639625  , -0.8020551 ,  0.4553566 ,  2.1744652 ,\n",
              "         -0.75405854,  0.39452946, -1.1817982 ,  1.0604678 ],\n",
              "        [ 0.2066074 , -1.1322836 , -1.0079966 ,  1.0053984 ,\n",
              "         -0.9590478 ,  0.34747452, -0.24890092,  2.4411082 ],\n",
              "        [-0.84917396, -0.340451  , -0.79904044,  0.8343787 ,\n",
              "          1.8163544 ,  1.3977008 , -0.4473251 , -0.0801996 ]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 126
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NDwvDtJeUwnA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dropout = tf.keras.layers.Dropout(config.dropout)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GTSc3HkxUzHl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "enc_out = dropout(enc_embed)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "07h9QYTRU33t",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "385b2131-c51f-4d51-b97b-ca8c026c117d"
      },
      "source": [
        "# encoder layer 생성 config.n_layer 개수\n",
        "encoder_layers = [EncoderLayer(config, name=f\"encoder_layer_{i}\") for i in range(config.n_layer)]\n",
        "assert len(encoder_layers) == config.n_layer\n",
        "encoder_layers"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<__main__.EncoderLayer at 0x7f554009cb00>,\n",
              " <__main__.EncoderLayer at 0x7f55400c49e8>,\n",
              " <__main__.EncoderLayer at 0x7f554016a748>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 129
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I9KWtAdeVBcf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6598d53a-9796-4359-c0db-8b4899ae2e16"
      },
      "source": [
        "# layer를 순환하며 실행. 이전 layer 결과가 다음 layer 입력이 됨\n",
        "for encoder_layer in encoder_layers:\n",
        "    enc_out = encoder_layer(enc_out, enc_self_mask)\n",
        "enc_out"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 6, 8), dtype=float32, numpy=\n",
              "array([[[-0.0164199 ,  1.7443534 , -1.9569262 ,  0.805189  ,\n",
              "          0.29447836, -0.01110621, -0.51996946, -0.3395991 ],\n",
              "        [ 0.6596276 , -0.21063349, -0.84621406,  2.2521105 ,\n",
              "         -0.92478037, -0.92603725,  0.08337084, -0.08744389],\n",
              "        [ 0.44695544, -0.23582178, -0.8397596 ,  2.1305463 ,\n",
              "         -1.0522934 , -1.0793475 ,  0.4280118 ,  0.20170875],\n",
              "        [ 1.2166032 , -1.2724937 , -0.417235  ,  1.5618125 ,\n",
              "         -0.7329493 , -1.0821369 ,  0.7565194 , -0.03012013],\n",
              "        [ 0.819291  , -0.93700683, -0.43735653,  1.8818612 ,\n",
              "         -1.0563735 , -1.0849974 ,  0.61743873,  0.19714335],\n",
              "        [-0.02201808,  0.60482967, -1.72039   , -0.15700594,\n",
              "          1.844797  , -0.1131691 ,  0.53396606, -0.9710096 ]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 130
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qmgQm2xYVYnu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dec_out = dropout(dec_embed)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KhwNJjYuVbE-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6c5e8142-7bf6-42bf-85f6-df8ab6326c1f"
      },
      "source": [
        "# decoder layer 생성 config.n_layer 개수\n",
        "decoder_layers = [DecoderLayer(config, name=f\"decoder_layer_{i}\") for i in range(config.n_layer)]\n",
        "assert len(decoder_layers) == config.n_layer\n",
        "decoder_layers"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<__main__.DecoderLayer at 0x7f554009c7b8>,\n",
              " <__main__.DecoderLayer at 0x7f55401b7b38>,\n",
              " <__main__.DecoderLayer at 0x7f5540191cc0>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 132
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bmQ8m5JMVrjw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2ab4f543-1bb2-4924-c88f-abae28185331"
      },
      "source": [
        "for decoder_layer in decoder_layers:\n",
        "    dec_out = decoder_layer(dec_out, enc_out, dec_self_mask, ende_attn_mask)\n",
        "dec_out"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 8), dtype=float32, numpy=\n",
              "array([[[ 0.30285424, -0.94883025,  0.1538057 ,  0.08533417,\n",
              "          1.1990507 , -2.1416757 ,  0.8097739 ,  0.53968716],\n",
              "        [ 0.43946582, -1.0445919 ,  0.19191447, -0.01680876,\n",
              "          1.034282  , -2.106845  ,  0.8839409 ,  0.6186423 ],\n",
              "        [ 0.42350546, -1.0721103 ,  0.1601551 , -0.14043942,\n",
              "          1.0635351 , -2.0400596 ,  0.9408983 ,  0.66451526],\n",
              "        [ 0.2995789 , -1.0043011 ,  0.03578647, -0.15494531,\n",
              "          1.1705276 , -2.0160024 ,  0.98067087,  0.68868476],\n",
              "        [ 0.3538905 , -1.0061551 , -0.11797055, -0.07693058,\n",
              "          1.4073578 , -1.947783  ,  0.916879  ,  0.47071174]]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 133
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fin1ZkzlV0f5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8638a7db-cd26-4163-eb23-8a75c6e653a8"
      },
      "source": [
        "# 최종 softmax 이전 linear 실행\n",
        "logits = embedding(dec_out, mode=\"linear\")\n",
        "logits"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 10), dtype=float32, numpy=\n",
              "array([[[-0.06621161,  0.2626956 , -0.28878814, -0.3288377 ,\n",
              "         -1.2169118 ,  0.92771053,  0.47236478,  0.33521676,\n",
              "         -0.07798429,  0.16750902],\n",
              "        [-0.23472112,  0.33451465, -0.17469805, -0.21850187,\n",
              "         -1.389923  ,  0.9765578 ,  0.3945058 ,  0.29124185,\n",
              "         -0.08254419,  0.23275888],\n",
              "        [-0.21893947,  0.340652  , -0.18914412, -0.14528832,\n",
              "         -1.3932418 ,  0.92554075,  0.29794604,  0.161028  ,\n",
              "         -0.09069928,  0.12650466],\n",
              "        [-0.09967335,  0.35724464, -0.28541893, -0.13641341,\n",
              "         -1.2403265 ,  0.80971336,  0.29909956,  0.0529805 ,\n",
              "         -0.12083422, -0.02653166],\n",
              "        [ 0.21233742,  0.22691667, -0.37151024, -0.14081179,\n",
              "         -0.91757333,  0.94576293,  0.24414268,  0.05377041,\n",
              "         -0.21177752, -0.03143537]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 134
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fQDSL7Q1TrvJ",
        "colab_type": "text"
      },
      "source": [
        "### Class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SD4jefa2SIpS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Transformer(tf.keras.layers.Layer):\n",
        "    \"\"\"\n",
        "    Transformer Class\n",
        "    \"\"\"\n",
        "    def __init__(self, config, name=\"transformer\"):\n",
        "        \"\"\"\n",
        "        생성자\n",
        "        :param config: Config 객체\n",
        "        :param name: layer name\n",
        "        \"\"\"\n",
        "        super().__init__(name=name)\n",
        "\n",
        "        self.i_pad = config.i_pad\n",
        "        self.embedding = SharedEmbedding(config)\n",
        "        self.position = PositionalEmbedding(config)\n",
        "        \n",
        "        self.encoder_layers = [EncoderLayer(config, name=f\"encoder_layer_{i}\") for i in range(config.n_layer)]\n",
        "        self.decoder_layers = [DecoderLayer(config, name=f\"decoder_layer_{i}\") for i in range(config.n_layer)]\n",
        "\n",
        "        self.dropout = tf.keras.layers.Dropout(config.dropout)\n",
        "\n",
        "    def call(self, inputs):\n",
        "        \"\"\"\n",
        "        layer 실행\n",
        "        :param inputs: (enc_tokens, dec_tokens) tuple\n",
        "        :return logits: dec_tokens에 대한 다음 토큰 예측 결과 logits\n",
        "        \"\"\"\n",
        "        enc_tokens, dec_tokens = inputs\n",
        "\n",
        "        enc_self_mask = tf.keras.layers.Lambda(get_pad_mask, output_shape=(1, None), name='enc_self_mask')(enc_tokens, self.i_pad)\n",
        "        dec_self_mask = tf.keras.layers.Lambda(get_ahead_mask, output_shape=(None, None), name='dec_self_mask')(dec_tokens, self.i_pad)\n",
        "        ende_attn_mask = tf.keras.layers.Lambda(get_pad_mask, output_shape=(1, None), name='ende_attn_mask')(enc_tokens, self.i_pad)\n",
        "\n",
        "        enc_embed = self.get_embedding(enc_tokens)\n",
        "        dec_embed = self.get_embedding(dec_tokens)\n",
        "\n",
        "        enc_out = self.dropout(enc_embed)\n",
        "        for encoder_layer in self.encoder_layers:\n",
        "            enc_out = encoder_layer(enc_out, enc_self_mask)\n",
        "        \n",
        "        dec_out = self.dropout(dec_embed)\n",
        "        for decoder_layer in self.decoder_layers:\n",
        "            dec_out = decoder_layer(dec_out, enc_out, dec_self_mask, ende_attn_mask)\n",
        "\n",
        "        logits = self.embedding(dec_out, mode=\"linear\")\n",
        "        return logits\n",
        "    \n",
        "    def get_embedding(self, tokens):\n",
        "        \"\"\"\n",
        "        token embedding, position embedding lookup\n",
        "        :param tokens: 입력 tokens\n",
        "        :return embed: embedding 결과\n",
        "        \"\"\"\n",
        "        embed = self.embedding(tokens) + self.position(tokens)\n",
        "        return embed"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_N5l-XL5WAuP",
        "colab_type": "text"
      },
      "source": [
        "### Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ho09l7wwWC0Y",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c8cc45d1-8a4b-442b-d3ca-71bc31a6aa1d"
      },
      "source": [
        "# Transformer 기능 확인. 최종 결과가 (bs, n_seq(dec), n_vocab)\n",
        "transformer = Transformer(config)\n",
        "\n",
        "logits = transformer((enc_tokens, dec_tokens))\n",
        "assert logits.shape == (dec_tokens.shape[0], dec_tokens.shape[1], config.n_vocab)\n",
        "logits"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 5, 10), dtype=float32, numpy=\n",
              "array([[[-0.22803521,  0.12426312, -0.43943322, -0.80678964,\n",
              "         -0.06882856,  1.1942756 ,  0.9215124 ,  0.03392895,\n",
              "         -0.05843499, -0.3590299 ],\n",
              "        [-0.25559124,  0.2801765 , -0.20359847, -0.82338655,\n",
              "         -0.01472074,  1.2325627 ,  1.1135418 ,  0.55003893,\n",
              "          0.13010341, -0.5705525 ],\n",
              "        [-0.12270705,  0.05449368, -0.467516  , -0.8853946 ,\n",
              "         -0.16956867,  1.1342182 ,  0.79351115, -0.20254382,\n",
              "         -0.08344322, -0.09620897],\n",
              "        [-0.03693207,  0.12546481, -0.36338612, -0.92431194,\n",
              "         -0.14594169,  1.0637096 ,  0.8613004 , -0.13289632,\n",
              "         -0.0784255 , -0.2210991 ],\n",
              "        [ 0.05455402,  0.12219068, -0.35891038, -0.9438473 ,\n",
              "         -0.17226166,  0.9925857 ,  0.84797823, -0.20073786,\n",
              "         -0.06487098, -0.2746322 ]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 136
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tOZoGy998QRV",
        "colab_type": "text"
      },
      "source": [
        "# Transformer Train (Songys)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SY0szhZ4cWzM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 위에서 Q, K, V의 K와 중복으로 인한 에러 수정\n",
        "import tensorflow.keras.backend as K"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Itc2MqpAEmex",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def print_file(filename, count=5):\n",
        "    \"\"\"\n",
        "    파일 내용 출력\n",
        "    :param filename: 파일 이름\n",
        "    :param count: 출력 라인 수\n",
        "    \"\"\"\n",
        "    with open(filename) as f:\n",
        "        for i, line in enumerate(f):\n",
        "            if count <= i:\n",
        "                break\n",
        "            print(line.strip())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "34abQnr98r7B",
        "colab_type": "text"
      },
      "source": [
        "## Download Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GFwVgQYX8uG3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "34788cd3-ef1f-440e-fad3-02bf01e83a65"
      },
      "source": [
        "# 다운로드\n",
        "!wget https://raw.githubusercontent.com/songys/Chatbot_data/master/ChatbotData%20.csv"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-08-23 13:14:08--  https://raw.githubusercontent.com/songys/Chatbot_data/master/ChatbotData%20.csv\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 889842 (869K) [text/plain]\n",
            "Saving to: ‘ChatbotData .csv’\n",
            "\n",
            "\rChatbotData .csv      0%[                    ]       0  --.-KB/s               \rChatbotData .csv    100%[===================>] 868.99K  --.-KB/s    in 0.05s   \n",
            "\n",
            "2020-08-23 13:14:09 (18.8 MB/s) - ‘ChatbotData .csv’ saved [889842/889842]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OtLSGXP-8quQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 이름변경 (원본 파일 이름에 공백 제거)\n",
        "!mv ChatbotData\\ .csv ChatbotData.csv"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U2hN4yQGDoHB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a60e2a98-06d6-4a05-f0f1-28fb68e81355"
      },
      "source": [
        "print_file(\"ChatbotData.csv\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Q,A,label\n",
            "12시 땡!,하루가 또 가네요.,0\n",
            "1지망 학교 떨어졌어,위로해 드립니다.,0\n",
            "3박4일 놀러가고 싶다,여행은 언제나 좋죠.,0\n",
            "3박4일 정도 놀러가고 싶다,여행은 언제나 좋죠.,0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S3266YeXD2Yw",
        "colab_type": "text"
      },
      "source": [
        "## Create Vocab"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kt-vE5irD-aI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b29dee5f-5206-454d-d0c1-ff3b4b20a8a4"
      },
      "source": [
        "# head=0 첫벗째 줄이 head\n",
        "chatbot_data = pd.read_csv(\"ChatbotData.csv\", header=0)\n",
        "chatbot_data.head(10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Q</th>\n",
              "      <th>A</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>12시 땡!</td>\n",
              "      <td>하루가 또 가네요.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1지망 학교 떨어졌어</td>\n",
              "      <td>위로해 드립니다.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3박4일 놀러가고 싶다</td>\n",
              "      <td>여행은 언제나 좋죠.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3박4일 정도 놀러가고 싶다</td>\n",
              "      <td>여행은 언제나 좋죠.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>PPL 심하네</td>\n",
              "      <td>눈살이 찌푸려지죠.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>SD카드 망가졌어</td>\n",
              "      <td>다시 새로 사는 게 마음 편해요.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>SD카드 안돼</td>\n",
              "      <td>다시 새로 사는 게 마음 편해요.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>SNS 맞팔 왜 안하지ㅠㅠ</td>\n",
              "      <td>잘 모르고 있을 수도 있어요.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>SNS 시간낭비인 거 아는데 매일 하는 중</td>\n",
              "      <td>시간을 정하고 해보세요.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>SNS 시간낭비인데 자꾸 보게됨</td>\n",
              "      <td>시간을 정하고 해보세요.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                         Q                   A  label\n",
              "0                   12시 땡!          하루가 또 가네요.      0\n",
              "1              1지망 학교 떨어졌어           위로해 드립니다.      0\n",
              "2             3박4일 놀러가고 싶다         여행은 언제나 좋죠.      0\n",
              "3          3박4일 정도 놀러가고 싶다         여행은 언제나 좋죠.      0\n",
              "4                  PPL 심하네          눈살이 찌푸려지죠.      0\n",
              "5                SD카드 망가졌어  다시 새로 사는 게 마음 편해요.      0\n",
              "6                  SD카드 안돼  다시 새로 사는 게 마음 편해요.      0\n",
              "7           SNS 맞팔 왜 안하지ㅠㅠ    잘 모르고 있을 수도 있어요.      0\n",
              "8  SNS 시간낭비인 거 아는데 매일 하는 중       시간을 정하고 해보세요.      0\n",
              "9        SNS 시간낭비인데 자꾸 보게됨       시간을 정하고 해보세요.      0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 142
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r5QPVFHbEE5P",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a4c5d359-35ac-422d-d76b-94428d6aef9e"
      },
      "source": [
        "# data 합치기\n",
        "corpus_set = pd.Series(chatbot_data['Q'].tolist() + chatbot_data['A'].tolist()).astype(str)\n",
        "corpus_set.head(10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0                     12시 땡!\n",
              "1                1지망 학교 떨어졌어\n",
              "2               3박4일 놀러가고 싶다\n",
              "3            3박4일 정도 놀러가고 싶다\n",
              "4                    PPL 심하네\n",
              "5                  SD카드 망가졌어\n",
              "6                    SD카드 안돼\n",
              "7             SNS 맞팔 왜 안하지ㅠㅠ\n",
              "8    SNS 시간낭비인 거 아는데 매일 하는 중\n",
              "9          SNS 시간낭비인데 자꾸 보게됨\n",
              "dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 143
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uWXTBj3hETZI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open(\"ChatbotData.txt\", \"w\") as f:\n",
        "    for line in corpus_set:\n",
        "        f.write(line.strip())\n",
        "        f.write(\"\\n\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IAvrv3rsEwhz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2f88d21e-7806-44e8-841d-98aadc98c3a0"
      },
      "source": [
        "print_file(\"ChatbotData.txt\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "12시 땡!\n",
            "1지망 학교 떨어졌어\n",
            "3박4일 놀러가고 싶다\n",
            "3박4일 정도 놀러가고 싶다\n",
            "PPL 심하네\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KlBr3ZMSE-Lz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train_sentencepiece(corpus, prefix, vocab_size=8000):\n",
        "    \"\"\"\n",
        "    sentencepiece를 이용해 vocab 학습\n",
        "    :param corpus: 학습할 말뭉치\n",
        "    :param prefix: 저장할 vocab 이름\n",
        "    :param vocab_size: vocab 개수\n",
        "    \"\"\"\n",
        "    spm.SentencePieceTrainer.train(\n",
        "        f\"--input={corpus} --model_prefix={prefix} --vocab_size={vocab_size + 7}\" +  # 7은 특수문자 개수\n",
        "        \" --model_type=bpe\" +\n",
        "        \" --max_sentence_length=999999\" +  # 문장 최대 길이\n",
        "        \" --pad_id=0 --pad_piece=[PAD]\" +  # pad token 및 id 지정\n",
        "        \" --unk_id=1 --unk_piece=[UNK]\" +  # unknown token 및 id 지정\n",
        "        \" --bos_id=2 --bos_piece=[BOS]\" +  # begin of sequence token 및 id 지정\n",
        "        \" --eos_id=3 --eos_piece=[EOS]\" +  # end of sequence token 및 id 지정\n",
        "        \" --user_defined_symbols=[SEP],[CLS],[MASK]\")  # 기타 추가 토큰 SEP: 4, CLS: 5, MASK: 6"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Eg6YJLDtFK_N",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# vocab 생성\n",
        "train_sentencepiece(f\"ChatbotData.txt\", f\"syngys\", vocab_size=8000)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WzrEXtlfFTDs",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8a3d4a98-6049-41f4-f4d5-209b2e6a5dcc"
      },
      "source": [
        "os.listdir(\".\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['.config',\n",
              " 'syngys.vocab',\n",
              " 'ChatbotData.txt',\n",
              " 'ChatbotData.csv',\n",
              " 'syngys.model',\n",
              " 'sample_data']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 148
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yREQozPuFV_U",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "75679c49-7e2c-4360-9585-8a995b94d324"
      },
      "source": [
        "# load vocab\n",
        "vocab = spm.SentencePieceProcessor()\n",
        "vocab.load(f\"syngys.model\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 149
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fAHHyXSKFb60",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c42ee45a-c712-482a-d548-99f355a565e1"
      },
      "source": [
        "# vocab 출력\n",
        "print(f\"len: {len(vocab)}\")\n",
        "for id in range(20):\n",
        "    print(f\"{id:2d}: {vocab.id_to_piece(id)}\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "len: 8007\n",
            " 0: [PAD]\n",
            " 1: [UNK]\n",
            " 2: [BOS]\n",
            " 3: [EOS]\n",
            " 4: [SEP]\n",
            " 5: [CLS]\n",
            " 6: [MASK]\n",
            " 7: 세요\n",
            " 8: ▁사\n",
            " 9: ▁좋\n",
            "10: ▁있\n",
            "11: ▁이\n",
            "12: ▁거\n",
            "13: 하는\n",
            "14: 보세요\n",
            "15: ▁그\n",
            "16: ▁나\n",
            "17: ▁아\n",
            "18: 예요\n",
            "19: ▁마\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NsFBGdvPFmRm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c87d75e4-6de0-4a00-c16c-3e2768b78793"
      },
      "source": [
        "pieces = vocab.encode_as_pieces(corpus_set[10])\n",
        "pieces"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['▁SNS', '보면', '▁나만', '▁빼고', '▁다', '▁행복해보여']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 151
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1-xIfzOMF8nO",
        "colab_type": "text"
      },
      "source": [
        "## Data Load"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "88FUzaPqGbT3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f633acbc-e07c-4d6b-ae93-24f7fbe297d0"
      },
      "source": [
        "# head=0 첫벗째 줄이 head\n",
        "chatbot_data = pd.read_csv(\"ChatbotData.csv\", header=0)\n",
        "chatbot_data.head(10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Q</th>\n",
              "      <th>A</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>12시 땡!</td>\n",
              "      <td>하루가 또 가네요.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1지망 학교 떨어졌어</td>\n",
              "      <td>위로해 드립니다.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3박4일 놀러가고 싶다</td>\n",
              "      <td>여행은 언제나 좋죠.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3박4일 정도 놀러가고 싶다</td>\n",
              "      <td>여행은 언제나 좋죠.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>PPL 심하네</td>\n",
              "      <td>눈살이 찌푸려지죠.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>SD카드 망가졌어</td>\n",
              "      <td>다시 새로 사는 게 마음 편해요.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>SD카드 안돼</td>\n",
              "      <td>다시 새로 사는 게 마음 편해요.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>SNS 맞팔 왜 안하지ㅠㅠ</td>\n",
              "      <td>잘 모르고 있을 수도 있어요.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>SNS 시간낭비인 거 아는데 매일 하는 중</td>\n",
              "      <td>시간을 정하고 해보세요.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>SNS 시간낭비인데 자꾸 보게됨</td>\n",
              "      <td>시간을 정하고 해보세요.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                         Q                   A  label\n",
              "0                   12시 땡!          하루가 또 가네요.      0\n",
              "1              1지망 학교 떨어졌어           위로해 드립니다.      0\n",
              "2             3박4일 놀러가고 싶다         여행은 언제나 좋죠.      0\n",
              "3          3박4일 정도 놀러가고 싶다         여행은 언제나 좋죠.      0\n",
              "4                  PPL 심하네          눈살이 찌푸려지죠.      0\n",
              "5                SD카드 망가졌어  다시 새로 사는 게 마음 편해요.      0\n",
              "6                  SD카드 안돼  다시 새로 사는 게 마음 편해요.      0\n",
              "7           SNS 맞팔 왜 안하지ㅠㅠ    잘 모르고 있을 수도 있어요.      0\n",
              "8  SNS 시간낭비인 거 아는데 매일 하는 중       시간을 정하고 해보세요.      0\n",
              "9        SNS 시간낭비인데 자꾸 보게됨       시간을 정하고 해보세요.      0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 152
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q5nDXAxSF77s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def load_data(vocab, dataframe):\n",
        "    \"\"\"\n",
        "    학습에 필요한 데이터를 로드\n",
        "    :param vocab: vocab\n",
        "    :param dataframe: pandas dataframe\n",
        "    :return enc_inputs: encoder inputs\n",
        "    :return dec_inputs: decoder inputs\n",
        "    :return labels: labels\n",
        "    \"\"\"\n",
        "    total = len(dataframe)\n",
        "\n",
        "    # 데이터 미리 생성\n",
        "    enc_tokens = []\n",
        "    dec_tokens = []\n",
        "    labels = []\n",
        "\n",
        "    # 라인단위로 데이터 생성\n",
        "    for i, row in tqdm(dataframe.iterrows(), total=len(dataframe)):\n",
        "        # Q, A 모두 string 인 경우만 처리\n",
        "        if type(row[\"Q\"]) == str and type(row[\"A\"]) == str:\n",
        "            # qestion을 id 형태로 변경\n",
        "            q_id = vocab.encode_as_ids(row[\"Q\"])\n",
        "            # answer를 id 형태로 변경\n",
        "            a_id = vocab.encode_as_ids(row[\"A\"])\n",
        "\n",
        "            # enc_token: <question tokens>, [PAD] tokens\n",
        "            enc_tokens.append(q_id)\n",
        "            # dec_token: [BOS], <answer tokens>, [PAD] tokens\n",
        "            dec_tokens.append([vocab.bos_id()] + a_id)\n",
        "            # label: <answer tokens>, [EOS], [PAD] tokens\n",
        "            labels.append(a_id + [vocab.eos_id()])\n",
        "    \n",
        "    enc_len = max([len(tokens) for tokens in enc_tokens])\n",
        "    dec_len = max([len(tokens) for tokens in dec_tokens])\n",
        "    label_len = max([len(tokens) for tokens in labels])\n",
        "\n",
        "    # 데이터 미리 생성\n",
        "    np_enc_tokens = np.zeros((total, enc_len), np.int)\n",
        "    np_dec_tokens = np.zeros((total, dec_len), np.int)\n",
        "    np_labels = np.zeros((total, label_len), np.int)\n",
        "\n",
        "    for i, (enc_token, dec_token, label) in enumerate(zip(enc_tokens, dec_tokens, labels)):\n",
        "        np_enc_tokens[i,:len(enc_token)] = enc_token\n",
        "        np_dec_tokens[i,:len(dec_token)] = dec_token\n",
        "        np_labels[i,:len(label)] = label\n",
        "\n",
        "    return (np_enc_tokens, np_dec_tokens), np_labels"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SUvENcJBGffw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "referenced_widgets": [
            "b7e44e68f53c4c869dcfbcdd1e745143",
            "d004b3226eaa4eed8cf18901fde5fb8d",
            "c374c11d8f4b4ac6bbaa50abc233ceb8",
            "d25a3e30f3584961afdd0ebfd7009d99",
            "8d0bf43a347e4bada53a1d448fe9c4d0",
            "2f9d1ec2fa6947ceb8ba53a10fdb314c",
            "ba2f186f0252446cbcdeb0253939b249",
            "029631a37ac343ab946110e0a0939fa4"
          ]
        },
        "outputId": "fc27a65b-2841-44d8-d624-62e0dcf8e8eb"
      },
      "source": [
        "train_inputs, train_labels = load_data(vocab, chatbot_data)\n",
        "# 마지막 출력. 마지막이 0이 아니면 끝까지 잘 처리 된 것임\n",
        "train_inputs[0][-1], train_inputs[1][-1], train_labels[-1]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b7e44e68f53c4c869dcfbcdd1e745143",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=11823.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([2475,  215, 4315,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0]),\n",
              " array([   2,  195, 7117, 7115, 3068,  392, 1475,  632, 6923,    0,    0,\n",
              "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "           0,    0,    0,    0,    0,    0,    0,    0]),\n",
              " array([ 195, 7117, 7115, 3068,  392, 1475,  632, 6923,    3,    0,    0,\n",
              "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "           0,    0,    0,    0,    0,    0,    0,    0]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 154
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xclGDmnmDCUY",
        "colab_type": "text"
      },
      "source": [
        "## Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R_9KQmbJDDyV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_pad_mask(tokens, i_pad=0):\n",
        "    \"\"\"\n",
        "    pad mask 계산하는 함수\n",
        "    :param tokens: tokens (bs, n_seq)\n",
        "    :param i_pad: id of pad\n",
        "    :return mask: pad mask (pad: 1, other: 0)\n",
        "    \"\"\"\n",
        "    mask = tf.cast(tf.math.equal(tokens, i_pad), tf.float32)\n",
        "    mask = tf.expand_dims(mask, axis=1)\n",
        "    return mask\n",
        "\n",
        "\n",
        "def get_ahead_mask(tokens, i_pad=0):\n",
        "    \"\"\"\n",
        "    ahead mask 계산하는 함수\n",
        "    :param tokens: tokens (bs, n_seq)\n",
        "    :param i_pad: id of pad\n",
        "    :return mask: ahead and pad mask (ahead or pad: 1, other: 0)\n",
        "    \"\"\"\n",
        "    n_seq = tf.shape(tokens)[1]\n",
        "    ahead_mask = 1 - tf.linalg.band_part(tf.ones((n_seq, n_seq)), -1, 0)\n",
        "    ahead_mask = tf.expand_dims(ahead_mask, axis=0)\n",
        "    pad_mask = get_pad_mask(tokens, i_pad)\n",
        "    mask = tf.maximum(ahead_mask, pad_mask)\n",
        "    return mask\n",
        "\n",
        "\n",
        "def kernel_initializer(stddev=0.02):\n",
        "    \"\"\"\n",
        "    parameter initializer 생성\n",
        "    :param stddev: 생성할 랜덤 변수의 표준편차\n",
        "    \"\"\"\n",
        "    return tf.keras.initializers.TruncatedNormal(stddev=stddev)\n",
        "\n",
        "\n",
        "class Config(dict):\n",
        "    \"\"\"\n",
        "    json을 config 형태로 사용하기 위한 Class\n",
        "    :param dict: config dictionary\n",
        "    \"\"\"\n",
        "    __getattr__ = dict.__getitem__\n",
        "    __setattr__ = dict.__setitem__\n",
        "\n",
        "    @classmethod\n",
        "    def load(cls, file):\n",
        "        \"\"\"\n",
        "        file에서 Config를 생성 함\n",
        "        :param file: filename\n",
        "        \"\"\"\n",
        "        with open(file, 'r') as f:\n",
        "            config = json.loads(f.read())\n",
        "            return Config(config)\n",
        "\n",
        "\n",
        "class SharedEmbedding(tf.keras.layers.Layer):\n",
        "    \"\"\"\n",
        "    Weighed Shaed Embedding Class\n",
        "    \"\"\"\n",
        "    def __init__(self, config, name=\"weight_shared_embedding\"):\n",
        "        \"\"\"\n",
        "        생성자\n",
        "        :param config: Config 객체\n",
        "        :param name: layer name\n",
        "        \"\"\"\n",
        "        super().__init__(name=name)\n",
        "\n",
        "        self.n_vocab = config.n_vocab\n",
        "        self.d_model = config.d_model\n",
        "    \n",
        "    def build(self, input_shape):\n",
        "        \"\"\"\n",
        "        shared weight 생성\n",
        "        :param input_shape: Tensor Shape (not used)\n",
        "        \"\"\"\n",
        "        with tf.name_scope(\"shared_embedding_weight\"):\n",
        "            self.shared_weights = self.add_weight(\n",
        "                \"weights\",\n",
        "                shape=[self.n_vocab, self.d_model],\n",
        "                initializer=kernel_initializer(stddev=self.d_model ** -0.5)\n",
        "            )\n",
        "\n",
        "    def call(self, inputs, mode=\"embedding\"):\n",
        "        \"\"\"\n",
        "        layer 실행\n",
        "        :param inputs: 입력\n",
        "        :param mode: 실행 모드\n",
        "        :return: embedding or linear 실행 결과\n",
        "        \"\"\"\n",
        "        # mode가 embedding일 경우 embedding lookup 실행\n",
        "        if mode == \"embedding\":\n",
        "            return self._embedding(inputs)\n",
        "        # mode가 linear일 경우 linear 실행\n",
        "        elif mode == \"linear\":\n",
        "            return self._linear(inputs)\n",
        "        # mode가 기타일 경우 오류 발생\n",
        "        else:\n",
        "            raise ValueError(f\"mode {mode} is not valid.\")\n",
        "    \n",
        "    def _embedding(self, inputs):\n",
        "        \"\"\"\n",
        "        embedding lookup\n",
        "        :param inputs: 입력\n",
        "        \"\"\"\n",
        "        embed = tf.gather(self.shared_weights, tf.cast(inputs, tf.int32))\n",
        "        embed *= self.d_model ** 0.5\n",
        "        return embed\n",
        "\n",
        "    def _linear(self, inputs):  # (bs, n_seq, d_model)\n",
        "        \"\"\"\n",
        "        linear 실행\n",
        "        :param inputs: 입력\n",
        "        \"\"\"\n",
        "        n_batch = tf.shape(inputs)[0]\n",
        "        n_seq = tf.shape(inputs)[1]\n",
        "        inputs = tf.reshape(inputs, [-1, self.d_model])  # (bs * n_seq, d_model)\n",
        "        outputs = tf.matmul(inputs, self.shared_weights, transpose_b=True)\n",
        "        outputs = tf.reshape(outputs, [n_batch, n_seq, self.n_vocab])  # (bs, n_seq, n_vocab)\n",
        "        return outputs\n",
        "\n",
        "\n",
        "class PositionalEmbedding(tf.keras.layers.Layer):\n",
        "    \"\"\"\n",
        "    Positional Embedding Class\n",
        "    \"\"\"\n",
        "    def __init__(self, config, name=\"position_embedding\"):\n",
        "        \"\"\"\n",
        "        생성자\n",
        "        :param config: Config 객체\n",
        "        :param name: layer name\n",
        "        \"\"\"\n",
        "        super().__init__(name=name)\n",
        "        \n",
        "        pos_encoding = PositionalEmbedding.get_sinusoid_encoding(config.n_seq, config.d_model)\n",
        "        self.embedding = tf.keras.layers.Embedding(config.n_seq, config.d_model, trainable=False, weights=[pos_encoding])\n",
        "\n",
        "    def call(self, inputs):\n",
        "        \"\"\"\n",
        "        layer 실행\n",
        "        :param inputs: 입력\n",
        "        :return embed: positional embedding lookup 결과\n",
        "        \"\"\"\n",
        "        position = tf.cast(tf.math.cumsum(tf.ones_like(inputs), axis=1, exclusive=True), tf.int32)\n",
        "        embed = self.embedding(position)\n",
        "        return embed\n",
        "\n",
        "    @staticmethod\n",
        "    def get_sinusoid_encoding(n_seq, d_model):\n",
        "        \"\"\"\n",
        "        sinusoid encoding 생성\n",
        "        :param n_seq: sequence number\n",
        "        :param n_seq: model hidden dimension\n",
        "        :return: positional encoding table\n",
        "        \"\"\"\n",
        "        angles = [np.power(10000, 2 * (i_ang // 2) / d_model) for i_ang in range(d_model)]\n",
        "        pos_encoding = np.array([[pos / angle for angle in angles] for pos in range(n_seq)])\n",
        "        pos_encoding[:, 0::2] = np.sin(pos_encoding[:, 0::2])\n",
        "        pos_encoding[:, 1::2] = np.cos(pos_encoding[:, 1::2])\n",
        "        return tf.cast(pos_encoding, tf.float32)\n",
        "\n",
        "\n",
        "class ScaleDotProductAttention(tf.keras.layers.Layer):\n",
        "    \"\"\"\n",
        "    Scale Dot Product Attention Class\n",
        "    \"\"\"\n",
        "    def __init__(self, name=\"scale_dot_product_attention\"):\n",
        "        \"\"\"\n",
        "        생성자\n",
        "        :param name: layer name\n",
        "        \"\"\"\n",
        "        super().__init__(name=name)\n",
        "\n",
        "    def call(self, Q, K, V, attn_mask):\n",
        "        \"\"\"\n",
        "        layer 실행\n",
        "        :param Q: Q value\n",
        "        :param K: K value\n",
        "        :param V: V value\n",
        "        :param attn_mask: 실행 모드\n",
        "        :return attn_out: attention 실행 결과\n",
        "        \"\"\"\n",
        "        attn_score = tf.matmul(Q, K, transpose_b=True)\n",
        "        scale = tf.math.sqrt(tf.cast(tf.shape(K)[-1], tf.float32))\n",
        "        attn_scale = tf.math.divide(attn_score, scale)\n",
        "        attn_scale -= 1.e9 * attn_mask\n",
        "        attn_prob = tf.nn.softmax(attn_scale, axis=-1)\n",
        "        attn_out = tf.matmul(attn_prob, V)\n",
        "        return attn_out\n",
        "\n",
        "\n",
        "class MultiHeadAttention(tf.keras.layers.Layer):\n",
        "    \"\"\"\n",
        "    Multi Head Attention Class\n",
        "    \"\"\"\n",
        "    def __init__(self, config, name=\"multi_head_attention\"):\n",
        "        \"\"\"\n",
        "        생성자\n",
        "        :param config: Config 객체\n",
        "        :param name: layer name\n",
        "        \"\"\"\n",
        "        super().__init__(name=name)\n",
        "\n",
        "        self.d_model = config.d_model\n",
        "        self.n_head = config.n_head\n",
        "        self.d_head = config.d_head\n",
        "\n",
        "        # Q, K, V input dense layer\n",
        "        self.W_Q = tf.keras.layers.Dense(config.n_head * config.d_head)\n",
        "        self.W_K = tf.keras.layers.Dense(config.n_head * config.d_head)\n",
        "        self.W_V = tf.keras.layers.Dense(config.n_head * config.d_head)\n",
        "        # Scale Dot Product Attention class\n",
        "        self.attention = ScaleDotProductAttention(name=\"self_attention\")\n",
        "        # output dense layer\n",
        "        self.W_O = tf.keras.layers.Dense(config.d_model)\n",
        "\n",
        "    def call(self, Q, K, V, attn_mask):\n",
        "        \"\"\"\n",
        "        layer 실행\n",
        "        :param Q: Q value\n",
        "        :param K: K value\n",
        "        :param V: V value\n",
        "        :param attn_mask: 실행 모드\n",
        "        :return attn_out: attention 실행 결과\n",
        "        \"\"\"\n",
        "        # reshape Q, K, V, attn_mask\n",
        "        batch_size = tf.shape(Q)[0]\n",
        "        Q_m = tf.transpose(tf.reshape(self.W_Q(Q), [batch_size, -1, self.n_head, self.d_head]), [0, 2, 1, 3])  # (bs, n_head, Q_len, d_head)\n",
        "        K_m = tf.transpose(tf.reshape(self.W_K(K), [batch_size, -1, self.n_head, self.d_head]), [0, 2, 1, 3])  # (bs, n_head, K_len, d_head)\n",
        "        V_m = tf.transpose(tf.reshape(self.W_V(V), [batch_size, -1, self.n_head, self.d_head]), [0, 2, 1, 3])  # (bs, n_head, K_len, d_head)\n",
        "        attn_mask_m = tf.expand_dims(attn_mask, axis=1)\n",
        "        # Scale Dot Product Attention with multi head Q, K, V, attn_mask\n",
        "        attn_out = self.attention(Q_m, K_m, V_m, attn_mask_m)  # (bs, n_head, Q_len, d_head)\n",
        "        # transpose and liner\n",
        "        attn_out_m = tf.transpose(attn_out, perm=[0, 2, 1, 3])  # (bs, Q_len, n_head, d_head)\n",
        "        attn_out = tf.reshape(attn_out_m, [batch_size, -1, config.n_head * config.d_head])  # (bs, Q_len, d_model)\n",
        "        attn_out = self.W_O(attn_out) # (bs, Q_len, d_model)\n",
        "\n",
        "        return attn_out\n",
        "\n",
        "\n",
        "class PositionWiseFeedForward(tf.keras.layers.Layer):\n",
        "    \"\"\"\n",
        "    Position Wise Feed Forward Class\n",
        "    \"\"\"\n",
        "    def __init__(self, config, name=\"feed_forward\"):\n",
        "        \"\"\"\n",
        "        생성자\n",
        "        :param config: Config 객체\n",
        "        :param name: layer name\n",
        "        \"\"\"\n",
        "        super().__init__(name=name)\n",
        "\n",
        "        self.W_1 = tf.keras.layers.Dense(config.d_ff, activation=tf.nn.relu)\n",
        "        self.W_2 = tf.keras.layers.Dense(config.d_model)\n",
        "\n",
        "    def call(self, inputs):\n",
        "        \"\"\"\n",
        "        layer 실행\n",
        "        :param inputs: inputs\n",
        "        :return ff_val: feed forward 실행 결과\n",
        "        \"\"\"\n",
        "        ff_val = self.W_2(self.W_1(inputs))\n",
        "        return ff_val\n",
        "\n",
        "\n",
        "class EncoderLayer(tf.keras.layers.Layer):\n",
        "    \"\"\"\n",
        "    Encoder Layer Class\n",
        "    \"\"\"\n",
        "    def __init__(self, config, name=\"encoder_layer\"):\n",
        "        \"\"\"\n",
        "        생성자\n",
        "        :param config: Config 객체\n",
        "        :param name: layer name\n",
        "        \"\"\"\n",
        "        super().__init__(name=name)\n",
        "\n",
        "        self.self_attention = MultiHeadAttention(config)\n",
        "        self.norm1 = tf.keras.layers.LayerNormalization(epsilon=config.layernorm_epsilon)\n",
        "\n",
        "        self.ffn = PositionWiseFeedForward(config)\n",
        "        self.norm2 = tf.keras.layers.LayerNormalization(epsilon=config.layernorm_epsilon)\n",
        "\n",
        "        self.dropout = tf.keras.layers.Dropout(config.dropout)\n",
        " \n",
        "    def call(self, enc_embed, self_mask):\n",
        "        \"\"\"\n",
        "        layer 실행\n",
        "        :param enc_embed: enc_embed 또는 이전 EncoderLayer의 출력\n",
        "        :param self_mask: enc_tokens의 pad mask\n",
        "        :return enc_out: EncoderLayer 실행 결과\n",
        "        \"\"\"\n",
        "        self_attn_val = self.self_attention(enc_embed, enc_embed, enc_embed, self_mask)\n",
        "        norm1_val = self.norm1(enc_embed + self.dropout(self_attn_val))\n",
        "\n",
        "        ffn_val = self.ffn(norm1_val)\n",
        "        enc_out = self.norm2(norm1_val + self.dropout(ffn_val))\n",
        "\n",
        "        return enc_out\n",
        "\n",
        "\n",
        "class DecoderLayer(tf.keras.layers.Layer):\n",
        "    \"\"\"\n",
        "    Decoder Layer Class\n",
        "    \"\"\"\n",
        "    def __init__(self, config, name=\"decoder_layer\"):\n",
        "        \"\"\"\n",
        "        생성자\n",
        "        :param config: Config 객체\n",
        "        :param name: layer name\n",
        "        \"\"\"\n",
        "        super().__init__(name=name)\n",
        "\n",
        "        self.self_attention = MultiHeadAttention(config)\n",
        "        self.norm1 = tf.keras.layers.LayerNormalization(epsilon=config.layernorm_epsilon)\n",
        "\n",
        "        self.ende_attn = MultiHeadAttention(config)\n",
        "        self.norm2 = tf.keras.layers.LayerNormalization(epsilon=config.layernorm_epsilon)\n",
        "\n",
        "        self.ffn = PositionWiseFeedForward(config)\n",
        "        self.norm3 = tf.keras.layers.LayerNormalization(epsilon=config.layernorm_epsilon)\n",
        "\n",
        "        self.dropout = tf.keras.layers.Dropout(config.dropout)\n",
        "\n",
        "    def call(self, dec_embed, enc_out, self_mask, ende_mask):\n",
        "        \"\"\"\n",
        "        layer 실행\n",
        "        :param dec_embed: dec_embed 또는 이전 DecoderLayer의 출력\n",
        "        :param enc_out: 마지막 EncoderLayer의 출력\n",
        "        :param self_mask: dec_tokens의 ahead mask\n",
        "        :param ende_mask: enc_tokens의 pad mask\n",
        "        :return dec_out: DecoderLayer 실행 결과\n",
        "        \"\"\"\n",
        "        self_attn_val = self.self_attention(dec_embed, dec_embed, dec_embed, self_mask)\n",
        "        norm1_val = self.norm1(dec_embed + self.dropout(self_attn_val))\n",
        "\n",
        "        ende_attn_val = self.ende_attn(norm1_val, enc_out, enc_out, ende_mask)\n",
        "        norm2_val = self.norm2(norm1_val + self.dropout(ende_attn_val))\n",
        "\n",
        "        ffn_val = self.ffn(norm2_val)\n",
        "        dec_out = self.norm3(norm2_val + self.dropout(ffn_val))\n",
        "\n",
        "        return dec_out\n",
        "\n",
        "\n",
        "class Transformer(tf.keras.layers.Layer):\n",
        "    \"\"\"\n",
        "    Transformer Class\n",
        "    \"\"\"\n",
        "    def __init__(self, config, name=\"transformer\"):\n",
        "        \"\"\"\n",
        "        생성자\n",
        "        :param config: Config 객체\n",
        "        :param name: layer name\n",
        "        \"\"\"\n",
        "        super().__init__(name=name)\n",
        "\n",
        "        self.i_pad = config.i_pad\n",
        "        self.embedding = SharedEmbedding(config)\n",
        "        self.position = PositionalEmbedding(config)\n",
        "        \n",
        "        self.encoder_layers = [EncoderLayer(config, name=f\"encoder_layer_{i}\") for i in range(config.n_layer)]\n",
        "        self.decoder_layers = [DecoderLayer(config, name=f\"decoder_layer_{i}\") for i in range(config.n_layer)]\n",
        "\n",
        "        self.dropout = tf.keras.layers.Dropout(config.dropout)\n",
        "\n",
        "    def call(self, inputs):\n",
        "        \"\"\"\n",
        "        layer 실행\n",
        "        :param inputs: (enc_tokens, dec_tokens) tuple\n",
        "        :return logits: dec_tokens에 대한 다음 토큰 예측 결과 logits\n",
        "        \"\"\"\n",
        "        enc_tokens, dec_tokens = inputs\n",
        "\n",
        "        enc_self_mask = tf.keras.layers.Lambda(get_pad_mask, output_shape=(1, None), name='enc_self_mask')(enc_tokens, self.i_pad)\n",
        "        dec_self_mask = tf.keras.layers.Lambda(get_ahead_mask, output_shape=(None, None), name='dec_self_mask')(dec_tokens, self.i_pad)\n",
        "        ende_attn_mask = tf.keras.layers.Lambda(get_pad_mask, output_shape=(1, None), name='ende_attn_mask')(enc_tokens, self.i_pad)\n",
        "\n",
        "        enc_embed = self.get_embedding(enc_tokens)\n",
        "        dec_embed = self.get_embedding(dec_tokens)\n",
        "\n",
        "        enc_out = self.dropout(enc_embed)\n",
        "        for encoder_layer in self.encoder_layers:\n",
        "            enc_out = encoder_layer(enc_out, enc_self_mask)\n",
        "        \n",
        "        dec_out = self.dropout(dec_embed)\n",
        "        for decoder_layer in self.decoder_layers:\n",
        "            dec_out = decoder_layer(dec_out, enc_out, dec_self_mask, ende_attn_mask)\n",
        "\n",
        "        logits = self.embedding(dec_out, mode=\"linear\")\n",
        "        return logits\n",
        "    \n",
        "    def get_embedding(self, tokens):\n",
        "        \"\"\"\n",
        "        token embedding, position embedding lookup\n",
        "        :param tokens: 입력 tokens\n",
        "        :return embed: embedding 결과\n",
        "        \"\"\"\n",
        "        embed = self.embedding(tokens) + self.position(tokens)\n",
        "        return embed"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A2Ys1-pYDMFo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# model build\n",
        "def build_model(config):\n",
        "    \"\"\"\n",
        "    model build\n",
        "    :param config: Configuration\n",
        "    :return model\n",
        "    \"\"\"\n",
        "    enc_tokens = tf.keras.layers.Input((None,), name=\"enc_tokens\")\n",
        "    dec_tokens = tf.keras.layers.Input((None,), name=\"dec_tokens\")\n",
        "\n",
        "    logits = Transformer(config)((enc_tokens, dec_tokens))\n",
        "    outputs = tf.keras.layers.Softmax(name=\"lm\")(logits)\n",
        "\n",
        "    model = tf.keras.Model(inputs=(enc_tokens, dec_tokens), outputs=outputs)\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pd9SRdI7DNWt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5cae05f8-4b46-46c2-eb8f-8b65d9199e88"
      },
      "source": [
        "config = Config({\"d_model\": 256, \"n_head\": 4, \"d_head\": 64, \"dropout\": 0.1, \"d_ff\": 1024, \"layernorm_epsilon\": 0.001, \"n_layer\": 3, \"n_seq\": 256, \"n_vocab\": 0, \"i_pad\": 0})\n",
        "config.n_vocab = len(vocab)\n",
        "config.i_pad = vocab.pad_id()\n",
        "config"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'d_ff': 1024,\n",
              " 'd_head': 64,\n",
              " 'd_model': 256,\n",
              " 'dropout': 0.1,\n",
              " 'i_pad': 0,\n",
              " 'layernorm_epsilon': 0.001,\n",
              " 'n_head': 4,\n",
              " 'n_layer': 3,\n",
              " 'n_seq': 256,\n",
              " 'n_vocab': 8007}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 157
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tB-BlxJBKs4A",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "47298bac-4f03-4dc1-9de0-8e54bda4657b"
      },
      "source": [
        "model = build_model(config)\n",
        "tf.keras.utils.plot_model(model, 'model.png', show_shapes=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuMAAAEnCAYAAAAD7E2bAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nOzde1hU1f4/8Pcol2GQq3ETJbkogmJ5DUj0GFleQryL5fFHdTqoncNYdDIxCy1U0mNoardDdjRTQXtEU6yjRmolWkoQBQlJqaSgyH2UAdbvDx/m6wgMAwxsBt6v5+GP1qxZ67P3dj592OxZSyaEECAiIiIioo6W1EPqCIiIiIiIuisW40REREREEmExTkREREQkERbjREREREQSMbm34bvvvsOGDRukiIWIiAAkJSW1y7jM70RE0mosvze4M37p0iXs3bu3QwIiMmanT5/G6dOnpQ7DqFy+fJn5RYf2Pj/M70T6YX5vOeZ33XSdnwZ3xuu1150Zoq5i9uzZAPhZaYnExETMnTuX56wJ9eenvfH8E+nG/N5yzO+66crvfGaciIiIiEgiLMaJiIiIiCTCYpyIiIiISCIsxomIiIiIJMJinIiIiIhIIizGiSR2+PBh2NjY4ODBg1KH0iktXLgQMplM8zN//vwGfY4ePYply5ahrq4O06dPh5ubG+RyOVxdXREaGoqMjIwWzdncOAcOHEBcXBxqa2u13rd//36tWO+7777WHzgRGT3md92Y3+9gMU4kMSGE1CF0evb29khJSUFOTg4SEhK0Xnv99dexadMmREdHo66uDidPnsSnn36K4uJinDp1CiqVCmPHjkVBQYHe8zU3ztSpUyGXyxEcHIySkhLN+0JDQ3H58mWcOHECkydPNtjxE5FxYn5vHvM7i3EiyU2ZMgWlpaUICQmROhSoVCoEBgZKHUYDFhYWmDhxIgYOHAhzc3NN+9q1a7F7924kJibCysoKABAQEIAxY8ZAoVDA3d0dsbGxKC0txccff9yiOZsbR6lU4oEHHsDkyZNRU1MDAJDJZHB1dUVQUBAGDBhgkGMnIuPF/N485ncW40R0l4SEBBQWFkodhl5yc3OxYsUKrFy5EnK5HABgYmLS4M/BHh4eAIC8vDy9x9Z3nJiYGKSnpyM+Pr5Vx0BE1FGY39GicToyv7MYJ5LQqVOn4ObmBplMhs2bNwMAtm7dCktLSygUCiQnJ2PSpEmwtrZG3759sWvXLs17N23aBLlcDkdHRyxcuBAuLi6Qy+UIDAxEWlqapl9kZCTMzMzg7OysaXv++edhaWkJmUyG69evAwCWLFmCqKgo5OXlQSaTwcvLCwBw5MgRWFtbIzY2tiNOid42bdoEIQSmTp2qs59KpQIAWFtbt2m+xsaxs7PDuHHjEB8fzz9HE5EW5vfW6275ncU4kYTGjBmDb7/9Vqtt8eLFeOGFF6BSqWBlZYU9e/YgLy8PHh4eeO6556BWqwHcScLh4eGoqqqCUqlEfn4+zp07h5qaGkyYMAGXLl0CcCepzZkzR2uOLVu2YOXKlVpt8fHxCAkJgaenJ4QQyM3NBQDNl1jq6ura5Ry01qFDh+Dt7Q2FQqGz35kzZwDcOddt0dQ4w4YNw5UrV/Djjz+2aXwi6lqY31uvu+V3FuNEnVhgYCCsra3h4OCAsLAwVFZW4o8//tDqY2JiAh8fH5ibm8PX1xdbt25FeXk5tm3bZpAYpkyZgrKyMqxYscIg4xlCZWUlLl68CE9Pzyb7XLt2Dbt374ZSqURAQECzd1haO079s4OZmZmtGp+Iuifm98Z1x/xu0q6jE5HBmJmZAYDmzklTRo4cCYVCgezs7I4ISxKFhYUQQui8axIQEIDKykrMmTMHb775JkxNTVs1V3Pj1Mdw7dq1Vo1PRMT8/n+6Y35nMU7UBZmbm6OoqEjqMNrNrVu3AEDrm/f3cnR0REJCAgYPHtymuZobx8LCQismIqL2xPze9fI7H1Mh6mLUajVKSkrQt29fqUNpN/UJ8t5NGe7m4OAAW1vbNs/V3DjV1dVaMRERtRfm9zu6Wn7nnXGiLiY1NRVCCPj7+2vaTExMmv3zpzFxdHSETCZDaWlpk30MteNdc+PUx+Dk5GSQ+YiImsL8fkdXy++8M05k5Orq6nDz5k3U1NQgIyMDS5YsgZubG8LDwzV9vLy8UFxcjP3790OtVqOoqAi///57g7Hs7e1RUFCA/Px8lJeXQ61WIyUlpdMtfaVQKODh4YHLly83+npubi6cnJwwd+7cBq+FhYXByckJ586da3YeXePUq4/Bz89Pz+iJiPTD/N5QV8zvLMaJJLR582aMGjUKALB06VKEhoZi69atePvttwEAQ4cOxW+//YYPP/wQUVFRAICJEyfiwoULmjFu3boFPz8/WFhYICgoCAMHDsRXX32l9bzd4sWLMX78eMybNw/e3t544403NH92CwgI0CyTtWjRIjg6OsLX1xeTJ09GcXFxh5yH1pgyZQqysrI068PeTdeasNXV1SgsLERycnKzc+iztuzZs2fh6uqKoUOHNtuXiLoP5vfW63b5Xdxjz549opFmIrrHrFmzxKxZsySNISIiQtjb20saQ0u0Jr9EREQIV1fXBu0XLlwQJiYmYseOHS0ar7a2VgQFBYmEhIQWva8x169fF3K5XKxfv77Ba0qlUvTu3btF47V3/mV+J9IP83vLMb/rpuP8JPLOOJGR0/Ull65CpVLhiy++wIULFzRfqPHy8sKqVauwatUqVFRU6DVObW0t9u/fj/LycoSFhbU5rpiYGDz44IOIjIwEcOdOS0FBAU6dOqXZVIOIqLWY37tHfmcxTkSdXnFxMSZOnIiBAwfimWee0bQvW7YMs2fPRlhYmM4v+9RLTU3Fvn37kJKS0uzObs3ZsGED0tPTcfjwYc3atMnJyXB1dUVQUBAOHTrUpvGJiLoD5ncW41rWrFkDGxsbyGQypKenSx2OXk6fPg0fHx/06NEDMpkMTk5OePPNN6UOS8u+ffvg4eEBmUwGmUwGZ2dnzJ8/X+qwjF50dDS2bduG0tJSuLu7Y+/evVKH1C7ee+89CCE0P5988onW67GxsYiMjMSaNWuaHSs4OBg7d+6Es7Nzm2JKTk7G7du3kZqaCjs7O037tGnTtGK9fv16m+bpjP72t7/BysrKqPIkwPzeXpjf2wfz+x3dJr+34JmWbmHXrl0CgDh//rzUobTI448/LgCImzdvSh1Kkzw9PYWNjY3UYRhMZ3im0Nh09/zSHGN5ZtxY86Sxxs383vGY31uO+V23bvfMuEqlQmBgoNRhdFs8/0TUXphfpMXzT2R4XbIYT0hIQGFhodRhdFs8/0Rdm0wmk2xu5hdp8fwTGZ5BivHa2lq89tprcHNzg4WFBYYOHYo9e/YAALZu3QpLS0soFAokJydj0qRJsLa2Rt++fbFr164GY+3YsQMjR46EXC6HpaUl+vfvjzfeeEPvWJYsWYKoqCjk5eVBJpPBy8sLwJ1vwW7YsAE+Pj4wNzeHnZ0dpk2bhuzsbJ3jXbt2Df3794eJiQkmTpxo8GP++uuvMXr0aCgUClhbW8PPzw9lZWUAgCNHjrR6MX59Y9i0aRPkcjkcHR2xcOFCuLi4QC6XIzAwEGlpaZp+kZGRMDMz03oW6/nnn4elpSVkMpnm2ammzn9LnTx5Er6+vrCxsYFcLoefnx+++OILAHeeWa1/PtHT0xPnz58HADz99NNQKBSwsbHBgQMHAOi+Tm+99RYUCgWsrKxQWFiIqKgouLq6Iicnp1UxE3VFQgisW7cO3t7eMDc3h42NDf71r3816Kfrs1aP+Z35HWB+J2qgBc+0NOmll14S5ubmYu/eveLmzZsiOjpa9OjRQ5w9e1YIIcTy5csFAHHs2DFRWloqCgsLRVBQkLC0tBTV1dWacd5++20BQKxZs0bcuHFDFBcXi/fff1889dRTLYpn5syZwtPTU6vttddeE2ZmZmLHjh2ipKREZGRkiOHDh4v77rtPXL16VdPv3mcKq6urxcyZM0VycrLBj7miokJYW1uLuLg4oVKpxNWrV8WMGTNEUVGREEKIzz//XFhZWYlVq1Y1e8yNPVOo73mPiIgQlpaW4ueffxa3bt0SWVlZYtSoUcLKykr88ccfmn5PPfWUcHJy0pp33bp1AoAm5qbOvxAte6YwKSlJxMTEiOLiYnHjxg3h7++vtabnzJkzRc+ePcWVK1e03vfkk0+KAwcOaP5b3+ukVCrFO++8I2bMmCF++eUXvWLkM4Utx2cKdeuMz4wvX75cyGQy8e9//1vcvHlTVFVViS1btjR49rq5zxrzO/N7Peb3ron5XTddz4y3uRhXqVRCoVCIsLAwTVtVVZUwNzcXixcvFkL83wdCpVJp+tQn89zcXCHEnaRoa2srxo8frzV+TU2NiI+P1zseIRomi6qqKtGrVy+tGIUQ4syZMwKAVjK8O1mr1Woxb948kZKS0i7H/NNPPwkA4vPPP2/R8TVGV7LWFYMQd5L1vUn07NmzAoBYuXKlpq0jk/W9Vq9eLQCIwsJCIYQQR48eFQDEm2++qelTWloqBgwYIGpqaoQQrb9O+mKybjkma906WzFeVVUlFAqFmDBhglb7vUVtc5815ve2YX5nfjcGzO+66SrGTdp6Zz0nJwdVVVUYMmSIps3CwgLOzs46/0RoZmYGAFCr1QCAjIwMlJSU4PHHH9fq17NnTyiVyjbFmJWVhYqKCowcOVKrfdSoUTAzM9P6c1292tpaPPnkk+jTp4/Wny8Bwx2zh4cHHB0dMX/+fCiVSoSHh6N///6tPUy93BtDU0aOHAmFQtHsn3k7Sv06n/UbIDzyyCMYOHAgPvroI0RHR0Mmk2H37t0ICwtDz549AbT+OrXE3r17JX1+1ljxnBmH3NxcVFVVITg4WGe/5j5rzO/M77owv3ctPGct1+ZivLKyEgDw6quv4tVXX9V6zcXFRe9x6p+js7W1bWtIDZSUlAAAevXq1eA1W1tblJeXN2j/xz/+gVu3buHAgQP4+9//Dl9fX81rhjpmCwsLHD9+HK+88gpiY2OxatUqzJkzB9u2bYOFhYXe47QXc3NzFBUVSTL3oUOHsG7dOmRlZaGsrKzB/1xkMhkWLlyIF198EceOHcOjjz6K7du3Y+fOnZo+hrpOuvj7++OFF14wyFjdwXfffYf4+PgGzxLTHfXnp7O4fPkyAMDBwUFnv+Y+a8zvzO93Y37vmpjfddOV39tcjNcn6bfffhtLlixp9Th9+vQBgHbZJKP+fwCNJeWSkhL07du3QfucOXPw17/+FUOGDMGCBQtw+vRpmJjcOV2GOmYAGDx4MA4ePIiioiJs2LABa9euxeDBg7FixYo2jdtWarW6yXPTHk6cOIEffvgBL7zwAv744w9Mnz4dM2bMwEcffYQ+ffrgnXfewcsvv6z1nvDwcERHR+M///kP+vXrB2tra9x///2a1w15nZrSt29fzJkzp13G7qri4+N5znToTMW4XC4HANy+fVtnv+Y+a/VfmmN+Z35nfu/amN91ayq/t3k1lX79+kEul7d5R7P+/fvD3t4eX375ZVtDamDIkCHo1asXvv/+e632tLQ0VFdXY8SIEQ3eM378eNx333344IMP8MMPP2jtemaoYy4oKMDPP/8M4E5iWbNmDYYPH65pk1JqaiqEEPD399e0mZiYNPvnz9b64YcfYGlpCQDIzMyEWq3G4sWL4eHhAblc3uifvezs7DB37lzs378f69evx3PPPaf1uqGuE1F3NWTIEPTo0QNff/21zn7NfdaY35nfmd+JmtbmYlwul+Ppp5/Grl27sHXrVpSVlaG2thaXL1/Gn3/+qfc45ubmiI6OxokTJxAZGYkrV66grq4O5eXlLU5e9vb2KCgoQH5+PsrLy9GzZ09ERUXhs88+wyeffIKysjJkZmZi0aJFcHFxQURERJNjTZ06FeHh4YiNjcUPP/xg0GMuKCjAwoULkZ2djerqapw/fx6///67JkGmpKS0eumrlqqrq8PNmzdRU1ODjIwMLFmyBG5ubggPD9f08fLyQnFxMfbv3w+1Wo2ioiL8/vvvDca69/zrSvBqtRrXrl1DamqqJlm7ubkBAI4ePYpbt27hwoULjT73CQCLFi3C7du38fnnnyMkJETrNUNdJ6LuysHBATNnzsTevXuRkJCAsrIyZGRk4IMPPtDq19xnjfmd+Z35nUiHFnzbs0m3b98WS5cuFW5ubsLExEQ4ODiImTNniqysLLFlyxahUCgEADFgwACRl5cnPvjgA2FtbS0AiPvvv1/8+uuvmrE2b94s/Pz8hFwuF3K5XAwbNkxs2bKlRfGcO3dO3H///cLCwkKMGTNGXL16VdTV1Yl169aJAQMGCFNTU2FnZyemT58ucnJyNO/bt2+fsLOzEwBE//79RWFhoSgrKxP9+vUTAESvXr3E9u3bDXbM+fn5IjAwUNjZ2YmePXuKPn36iOXLl2u+LX748GFhZWWl9Y3ye50+fVoMHjxY9OjRQwAQzs7OIjY2tkXnPSIiQpiamgpXV1dhYmIirK2txbRp00ReXp7WXDdu3BDjx48XcrlcuLu7i3/+85/iX//6lwAgvLy8NMtk3Xv+3333XeHp6SkA6Pz57LPPNHMtXbpU2NvbC1tbWzF79myxefNmAUB4enpqLcclhBDDhg0Ty5Yta/T86LpOcXFxwsLCQgAQ/fr1Ezt27ND3n5gQgt+2bw1+2163zraaihBClJeXi7/97W+id+/eolevXmLMmDHitddeEwBE3759xY8//iiE0P1Zq8f8zvwuBPN7V8X8rpuu1VRkQghxd3GemJiIuXPn4p5m6sIWLlyIpKQk3LhxQ+pQWmXKlCnYvHkz3N3dO3Te2bNnAwCSkpI6dF5jxvyiW3ufH57/7of5vXWY31uO+UU3HecnySA7cJLxq19Syhjc/WfRjIwMyOXyDk/URETGgvmdqHMzimI8Oztbsz2urp+wsDCpQ6UOsHTpUly4cAG//vornn766RZtp03GZ+HChVqf8/nz5zfoc/ToUSxbtgx1dXWYPn063NzcIJfL4erqitDQUGRkZLRozubGOXDgAOLi4hoUOfv379eK9b777mv9gXcTzO90N+b37oX5/Q6jKMYHDRoEIUSzP7t375Y6VKMTHR2Nbdu2obS0FO7u7ti7d6/UITVLoVBg0KBBePTRRxETE6O1RjB1Tfb29khJSUFOTg4SEhK0Xnv99dexadMmREdHo66uDidPnsSnn36K4uJinDp1CiqVCmPHjkVBQYHe8zU3ztSpUyGXyxEcHKxZ5xoAQkNDcfnyZZw4cQKTJ0822PF3Zczv7Yf5nYwB8zsM8wVOou6oM3zBp6qqSgQEBBjNHK3JLxEREcLV1bXR19asWSMGDhyo2e5arVaLJ554QqtP/bbosbGxes+p7ziRkZEiICBAqNXqBmMolUrRu3dvvecUonN+gZOoO2J+bznmd910fYHTKO6ME1HjEhISUFhYaPRztEZubi5WrFiBlStXajanMTExwcGDB7X6eXh4AADy8vL0HlvfcWJiYpCent6pNuohoq6B+b375HcW40QdSAiBDRs2wMfHB+bm5rCzs8O0adOQnZ2t6RMZGQkzMzM4Oztr2p5//nlYWlpCJpNpdjFcsmQJoqKikJeXB5lMBi8vL2zatAlyuRyOjo5YuHAhXFxcIJfLERgYqLWWb1vmAIAjR4502BrJTdm0aROEEJg6darOfiqVCgBgbW3dpvkaG8fOzg7jxo1DfHw8VxAg6uaY3w2nu+V3FuNEHSgmJgbLli3D8uXLUVhYiBMnTuDSpUsICgrCtWvXANxJQvduJ7xlyxasXLlSqy0+Ph4hISHw9PSEEAK5ubmIjIxEeHg4qqqqoFQqkZ+fj3PnzqGmpgYTJkzApUuX2jwH8H+rM9TV1Rnu5LTQoUOH4O3tDYVCobPfmTNnAABjxoxp03xNjTNs2DBcuXIFP/74Y5vGJyLjxvxuON0tv7MYJ+ogKpUKGzZswIwZMzB//nzY2NjAz88P7733Hq5fv95gV8O2MDEx0dyd8fX1xdatW1FeXo5t27YZZPwpU6agrKwMK1asMMh4LVVZWYmLFy/C09OzyT7Xrl3D7t27oVQqERAQ0OwdltaOM2DAAAB3tvkmou6J+d1wumN+N2nX0YlIIysrCxUVFRg5cqRW+6hRo2BmZtbkltCGMHLkSCgUCq0/lxqzwsJCCCF03jUJCAhAZWUl5syZgzfffBOmpqatmqu5cepjqL/zRUTdD/O74XTH/M5inKiD1C+R1KtXrwav2draory8vF3nNzc3R1FRUbvO0VFu3boF4M4xNcXR0REJCQkYPHhwm+ZqbhwLCwutmIio+2F+N5zumN/5mApRB7G1tQWARpNySUkJ+vbt225zq9Xqdp+jI9UnSF07Czo4OGjOeVs0N051dbVWTETU/TC/G053zO+8M07UQYYMGYJevXrh+++/12pPS0tDdXU1RowYoWkzMTHR2ha6rVJTUyGEgL+/f7vN0ZEcHR0hk8lQWlraZJ97l65qrebGqY/BycnJIPMRkfFhfjec7pjfeWecqIPI5XJERUXhs88+wyeffIKysjJkZmZi0aJFcHFxQUREhKavl5cXiouLsX//fqjVahQVFeH3339vMKa9vT0KCgqQn5+P8vJyTfKtq6vDzZs3UVNTg4yMDCxZsgRubm4IDw83yBwpKSmSLn2lUCjg4eGBy5cvN/p6bm4unJycMHfu3AavhYWFwcnJCefOnWt2Hl3j1KuPwc/PT8/oiairYX43nO6Y31mME3Wg119/HatXr8aqVatw3333Ydy4cejfvz9SU1NhaWmp6bd48WKMHz8e8+bNg7e3N9544w3Nn8kCAgI0S1gtWrQIjo6O8PX1xeTJk1FcXAzgzvNtfn5+sLCwQFBQEAYOHIivvvpK6xm8ts4htSlTpiArK0uzPuzddK0JW11djcLCQiQnJzc7hz5ry549exaurq4YOnRos32JqOtifjecbpffW7BdJxHdpTNsl9yYiIgIYW9vL3UYjTLkdskXLlwQJiYmYseOHS0ar7a2VgQFBYmEhIQWva8x169fF3K5XKxfv77BawbeLtkgmN+J9MP83nLM77rpOD+JvDNO1AXp+uKLMVKpVPjiiy9w4cIFzRdqvLy8sGrVKqxatQoVFRV6jVNbW4v9+/ejvLwcYWFhbY4rJiYGDz74ICIjIwHcudNSUFCAU6dOaTbQICIyJOb3xhlzfmcxTkSdXnFxMSZOnIiBAwfimWee0bQvW7YMs2fPRlhYmM4v+9RLTU3Fvn37kJKS0uzObs3ZsGED0tPTcfjwYc3atMnJyXB1dUVQUBAOHTrUpvGJiLoD5ncW40RdSnR0NLZt24bS0lK4u7tj7969UofUZu+99x6EEJqfTz75ROv12NhYREZGYs2aNc2OFRwcjJ07d8LZ2blNMSUnJ+P27dtITU2FnZ2dpn3atGlasV6/fr1N8xAR1WN+182Y8zuXNiTqQlavXo3Vq1dLHUaHe+yxx/DYY4912HyhoaEIDQ3tsPmIiJjfO4YU+Z13xomIiIiIJMJinIiIiIhIIizGiYiIiIgkwmKciIiIiEgiTX6BMzExsSPjIDI69dvkdsRnRQgBmUzW7vO0t++++w4A80tT6s9Pe+P57x66St6QQkfm966C+V03XfldJoT2fqCJiYmYO3duuwdFRESNE3ps09wazO9ERNJqJL8nNSjGiahz+eWXXxAZGYljx45h/vz5iIuLg4uLi9RhEVEndOLECfzzn/9EdnY2lixZghUrVqBXr15Sh0VETUviM+NEnZyPjw/+97//ITk5GadOncKAAQMQExOD27dvSx0aEXUSBQUFWLBgAf7yl7/AwcEB586dQ1xcHAtxIiPAYpzISISEhCArKwsrVqzAv//9b/j5+XHLdaJuTq1WY+PGjRg0aBBSU1Px8ccf4+jRoxg8eLDUoRGRnliMExkRCwsLLF26FL/88gv8/f3xxBNPICQkBL/99pvUoRFRBzt69CgeeOABREdH48UXX8Svv/6KBQsWSB0WEbUQi3EiI9S3b19s374dx48fR35+Pnx8fKBUKlFeXi51aETUznJzcxESEoIJEybA09MTWVlZiImJgVwulzo0ImoFFuNERmz8+PE4f/483nrrLfz3v/+Fj48Ptm/f3m6rcRCRdKqqqhATE4MhQ4YgNzcXR44cwcGDB9G/f3+pQyOiNmAxTmTkTExMoFQqkZeXh5kzZ+Lpp5/G+PHjkZGRIXVoRGQgBw8ehI+PDzZu3Ii4uDhkZmbi8ccflzosIjIAFuNEXUTv3r2xceNGnDlzBtXV1RgxYgQiIiJw/fp1qUMjolZKT0/H2LFjERoainHjxiEnJwdKpRImJk3u2UdERobFOFEXM2LECHzzzTdISEhAcnIyvL29sXHjRtTW1kodGhHp6ebNm1AqlRg5ciRUKhW+/fZbbN++HY6OjlKHRkQGxmKcqAuSyWRYsGABsrOz8dxzz+Hll1/G6NGj8c0330gdGhHpUFdXh+3bt8Pb2xuJiYnYunUr0tLS4O/vL3VoRNROWIwTdWG2trZYu3YtMjIy4ODggKCgIMyZMweXLl2SOjQiuseZM2cQEBCAv/3tb5g3bx6ys7Px97//HT168H/VRF0ZP+FE3YC3tzeOHDmC5ORknD17Fj4+PtzFk6iTqN8909/fH7169cL58+exceNG2NjYSB0aEXUAFuNE3UhISAh+/vlnvPTSS4iLi4Ofnx8OHz4sdVhE3VL97pk+Pj746quv8PHHH+PYsWPcPZOom2ExTtTNWFhYICYmBhcuXIC/vz+mTJnCXTyJOtjRo0fx4IMPYtmyZXjhhRdw4cIF7p5J1E2xGCfqpup38Tx27BguXryo2cWzoqJC6tCIuqy8vDzMmTMHEyZMgIeHB37++WfunknUzbEYJ+rmHnnkEa1dPAcNGoTt27dLHRZRl3L37pkZGRncPZOINFiMExFMTU2hVCqRnZ2NKVOmaHbxzMzMlDo0IqN38OBB+Pr6YuPGjYiJiUFGRgZ3zyQiDRbjRKTh7OyM999/H2lpabh16xaGD/2fgz8AACAASURBVB+OiIgI3LhxQ+rQiIxOdnY2HnvsMYSGhmLs2LHIzs7G0qVLYWZmJnVoRNSJsBgnogZGjhyJb7/9FgkJCdi/fz8GDhzIXTyJ9FS/e6afnx+Ki4vxzTffYPv27XBycpI6NCLqhFiME1Gj6nfxzMnJwfz58/HSSy9h9OjR+Pbbb6UOjahTqt89c9CgQdi5cyfWr1+v2ciHiKgpLMaJSCdbW1ts3LgRmZmZcHBwwJgxY7BgwQJcvXpV6tCIOo2zZ88iMDAQzz77LMLCwpCXlwelUsndM4moWcwSRKSXQYMGaXbxPHnyJLy8vBATE4Pq6mqpQyOSzJ9//okFCxbgoYcegkKh4O6ZRNRiLMaJqEVCQkKQlZWltYtnSkqK1GERdaj63TMHDRqk2T3z+PHjGDJkiNShEZGRYTFORC2mUCgQExODX3/9FQ899BAmT56MkJAQXLx4UerQiNrdsWPHMGzYMCxbtgyLFi3CL7/8wt0ziajVWIwTUav169cP27dvx9GjR3Hx4kUMHjwYr7zyCnfxpC6pfvfMRx99FO7u7sjKysLatWvRq1cvqUMjIiPGYpyI2iw4OBjnz5/HmjVr8O6778LHx4e7eFKXcffumT/++CNSUlJw8OBBuLu7Sx0aEXUBLMaJyCDu3sVz8uTJePrpp/HII4/gp59+kjo0olar3z1z/fr1WLp0KTIzMzFx4kSpwyKiLoTFOBEZlIuLC95//32cPn0aVVVVGDZsGJRKJUpLS6UOjUhv2dnZePzxxzW7Z+bl5SEmJoa7ZxKRwbEYJ6J2MWrUKM0unrt374anpyc2btyIuro6qUMjalJJSYlm98zr16/j1KlT3D2TiNoVi3Eiajc9evTAggULkJ2djaeeekqzi+d3330ndWhEWup3z/T29tbaPTMwMFDq0Iioi2MxTkTtzs7OTrOLZ+/evfHwww9jwYIFuHbtmtShEeH777/Hww8/jGeffRbTpk1DTk4OlEolevbsKXVoRNQNsBgnog4zaNAgfPHFF0hOTsaJEycwaNAgxMXFcRdPksTVq1cRERGBhx56CHK5HOfOncP777+P3r17Sx0aEXUjLMaJqMOFhITg559/hlKpRExMDIYOHYojR45IHRZ1E/W7Z3p7e+PQoUPYtm0bvvrqK/j5+UkdGhF1QyzGiUgS9bt4/vTTTxg6dCgmTZqEkJAQ5OfnSx0adWHHjx/X2j0zOzubu2cSkaRYjBORpDw9PZGYmIijR4/it99+g6+vL2JiYqBSqaQOjbqQS5cuYcGCBQgODubumUTUqbAYJ6JOITg4GOnp6VizZg3efvttDBw4kLt4UpupVCrExMRg4MCBSEtLw+HDh7l7JhF1KizGiajTuHsXz/HjxyM8PBzBwcF67eL5+eefd0CE1BmoVCocP3682X737p6ZkZGBSZMmdUCERET6YzFORJ2Oi4sLtm/fjrS0NFRUVGD48OE6d/E8ePAgpk6dij179nRwpNTRhBB46qmnMHfu3Cb/PeTk5GDixIkIDQ1FUFAQcnNzERMTA3Nz8w6OloioeSzGiajTGjVqFL777jv85z//wa5duxrdxfP27dv4xz/+AQD461//itOnT0sVLnWAV199FcnJySgpKcHKlSu1XispKcErr7yCoUOHoqioCCdPnsT27dvh7OwsUbRERM1jMU5EnVr9Lp45OTl46qmnEBUVhYceekhTdK9fvx5XrlyBEAJ1dXWYNGkS8vLyJI6a2sOePXuwZs0a1NXVoaamBps2bUJmZiaEENi+fTsGDRqEDz/8EG+99RbOnDmDhx9+WOqQiYiaJRNCCKmDICLSV3p6OiIjI/HNN99g3rx52LdvH27duqV53dTUFP3798fZs2dhY2MjYaRkSKdOncIjjzwCtVqtaTM1NcWwYcMghMD58+exePFirFy5Era2thJGSkTUIkm8M05ERuXBBx/E119/jR07duD7779HbW2t1utqtRr5+fmYPXs2ampqJIqSDOnixYsIDQ3VejwJuHOtz5w5A1NTU5w/fx4bN25kIU5ERofFOBEZHZlMBjc3N/z6669ad0rrqdVqHD9+HFFRURJER4ZUVlaGSZMmoaysrMEvXsCdx5guXrwIDw8PCaIjImo7FuNEZHRqa2sRERGBHj2aTmG1tbV45513sGXLlg6MjAyppqYG06ZNw2+//dbkXznq6upQVFSEdevWdXB0RESGwWKciIzOu+++i+zs7EbvlN5NCIHIyEh8+eWXHRQZGdI//vEPnDhxotG/ftytpqYGq1evRn5+fscERkRkQCzGicioXL9+HStWrAAAmJmZ6fWeGTNm4Oeff27PsMjA4uPj8cEHHzT7C5dMJoOZmRmqq6v5WBIRGSWupkJERqW2thbZ2dn44Ycf8MMPPyAtLQ0//vgjbt26hR49esDU1BS3b9/Weo+pqSkcHR1x7tw5ODo6ShQ56SslJQVPPPFEgy9s3l14CyFgZWWFwYMHY/To0RgxYgRGjBiBwYMHSxQ1EVGrJLEYJyKjV1tbi5ycHPzwww84d+6cpkCvqqpCjx490LNnT6jVaowePRonTpzgToydWEZGBvz9/aFSqWBqaqp5RMXBwQGjR4/G6NGjMXz4cAwfPhx9+vSROFoiojZjMU6di0wmkzoEIiLqwmbNmoWkpCSpwyCql2QidQRE91qyZAkCAgKkDoO6ICEErl69CltbW1hYWEgdTrubO3euUX2eioqKIJfLYWVlJVkMb7/9NgDghRdekCwGaj/115eoM2ExTp1OQEAA5syZI3UYREZv7ty5/Dy1UP0dU56zrol3xKkz4moqREREREQSYTFORERERCQRFuNERERERBJhMU5EREREJBEW40REREREEmExTkREOh0+fBg2NjY4ePCg1KF0ekePHsWyZcuabK+rq8P06dPh5uYGuVwOV1dXhIaGIiMjo0XzNDfOgQMHEBcXh9raWoMel9TxG/q4iDoDFuNERKQT94bTz+uvv45NmzYhOjq6yfa6ujqcPHkSn376KYqLi3Hq1CmoVCqMHTsWBQUFes/V3DhTp06FXC5HcHAwSkpKDHZcUsdvyOMi6ixYjBMRkU5TpkxBaWkpQkJCpA4FKpUKgYGBUofRwNq1a7F7924kJiZqbVrUWHtAQADGjBkDhUIBd3d3xMbGorS0FB9//HGL5mxuHKVSiQceeACTJ09GTU2NwY5L6vgNcVxEnQmLcSIiMhoJCQkoLCyUOgwtubm5WLFiBVauXAm5XK6z3cTEpMHjPh4eHgCAvLw8vefUd5yYmBikp6cjPj6+ZQfVyeNvy3ERdTYsxomIqEmnTp2Cm5sbZDIZNm/eDADYunUrLC0toVAokJycjEmTJsHa2hp9+/bFrl27NO/dtGkT5HI5HB0dsXDhQri4uEAulyMwMBBpaWmafpGRkTAzM4Ozs7Om7fnnn4elpSVkMhmuX78OAFiyZAmioqKQl5cHmUwGLy8vAMCRI0dgbW2N2NjYjjglDWzatAlCCEydOlWv9nupVCoAgLW1dZviaGwcOzs7jBs3DvHx8S1+3Kgzx9+W4yLqbFiMExFRk8aMGYNvv/1Wq23x4sV44YUXoFKpYGVlhT179iAvLw8eHh547rnnoFarAdwpssPDw1FVVQWlUon8/HycO3cONTU1mDBhAi5dugTgTtF37/bzW7ZswcqVK7Xa4uPjERISAk9PTwghkJubCwCaL/PV1dW1yzlozqFDh+Dt7Q2FQqFX+73OnDkD4M65boumxhk2bBiuXLmCH3/8sUXjdfb4W3tcRJ0Ni3EiImq1wMBAWFtbw8HBAWFhYaisrMQff/yh1cfExAQ+Pj4wNzeHr68vtm7divLycmzbts0gMUyZMgVlZWVYsWKFQcZricrKSly8eBGenp56td/t2rVr2L17N5RKJQICApq9A93acQYMGAAAyMzM1HtMY4i/NcdF1BmZSB0AERF1DWZmZgCguTPelJEjR0KhUCA7O7sjwmpXhYWFEEI0uHvcVPvdAgICUFlZiTlz5uDNN9+Eqalpq2Jobpz6GK5du6b3mMYQf2uOi6gzYjFOREQdztzcHEVFRVKH0Wa3bt0CcOd49Gm/m6OjIxISEjB48OA2xdDcOBYWFlox6cMY4m/NcRF1RnxMhYiIOpRarUZJSQn69u0rdShtVl8Q3rsJTVPtd3NwcICtrW2bY2hunOrqaq2Y9GEM8bfmuIg6I94ZJyKiDpWamgohBPz9/TVtJiYmzT7e0hk5OjpCJpOhtLRUr/a7GWpH0+bGqY/ByclJ7zGNIf7WHBdRZ8Q740RE1K7q6upw8+ZN1NTUICMjA0uWLIGbmxvCw8M1fby8vFBcXIz9+/dDrVajqKgIv//+e4Ox7O3tUVBQgPz8fJSXl0OtViMlJUWypQ0VCgU8PDxw+fJlvdrr5ebmwsnJCXPnzm3wWlhYGJycnHDu3Llm59c1Tr36GPz8/PQevzPH31w7kbFhMU5ERE3avHkzRo0aBQBYunQpQkNDsXXrVrz99tsAgKFDh+K3337Dhx9+iKioKADAxIkTceHCBc0Yt27dgp+fHywsLBAUFISBAwfiq6++0noeefHixRg/fjzmzZsHb29vvPHGG5rHDwICAjTLIC5atAiOjo7w9fXF5MmTUVxc3CHnQZcpU6YgKytLs052c+0AdK6NXV1djcLCQiQnJzc7tz5rbJ89exaurq4YOnRoi8bvrPE3105kdARRJwJA7NmzR+owiLqEzvB5ioiIEPb29pLG0BKzZs0Ss2bNatF7Lly4IExMTMSOHTv0am9ObW2tCAoKEgkJCS16X2OuX78u5HK5WL9+fYvH76zx62pvTmuuL1E7S+SdcSIiale6vgTYFXh5eWHVqlVYtWoVKioqmm3Xpba2Fvv370d5eTnCwsLaHFtMTAwefPBBREZGtnj8zhh/c+1ExojFOJGB3b59G0qlEs7OzlAoFDhy5IjUIbWbsLAwyGQyvX4+//xzqcPtVteGOtayZcswe/ZshIWFaX3psan2pqSmpmLfvn1ISUlpdufL5mzYsAHp6ek4fPiwZu3ulo7f2eLX1U5krFiMExnYv//9bxw5cgTZ2dmIj4/X+46Ssfryyy9RUlICtVqNP//8EwAwdepUVFdXo7KyEoWFhXjuueckjvKO7nZtpBYdHY1t27ahtLQU7u7u2Lt3r9QhtavY2FhERkZizZo1erU3Jjg4GDt37oSzs3ObYklOTsbt27eRmpoKOzu7No3fmeJvqp3ImMmE0OPbE0QdRCaTYc+ePZgzZ45e/VUqFYKDg/Htt9+2c2T6Gz16NAYMGICdO3dKHUq7mzdvHhISEjR3wK5evQoXFxeEhoZi//79mn7vv/8+XF1d8cQTT0gVKoDudW2Aln+eCJg9ezYAICkpSeJIqD3w+lInlMR1xsmoJSQkoLCwUOowtFy+fBm+vr5Sh9Ehdu3apVe/iIiIdo5EP93p2hARkXHgYypktJYsWYKoqCjk5eVBJpPBy8sLb731FhQKBaysrFBYWIioqCi4uroiJycHJ0+ehK+vL2xsbCCXy+Hn54cvvvgCALB161ZYWlpCoVAgOTkZkyZNgrW1Nfr27dug4Pz6668xevRoKBQKWFtbw8/PD2VlZfjf//4HLy8v/Pnnn/jvf/8LmUyGXr16AbizfNeGDRvg4+MDc3Nz2NnZYdq0acjOztaM21TsixYtgqWlJXr06IERI0bAyckJpqamsLS0xPDhwxEUFIR+/fpBLpfD1tYWL7/8sla8tbW1eO211+Dm5gYLCwsMHToUe/bs0TlnTk4Ojhw5YtC1m3ltDHdtiIioC5F2NRcibWjhUmwzZ84Unp6eWm3Lly8XAIRSqRTvvPOOmDFjhvjll19EUlKSiImJEcXFxeLGjRvC399f9O7du8H7jh07JkpLS0VhYaEICgoSlpaWorq6WgghREVFhbC2thZxcXFCpVKJq1evihkzZoiioiLNOE5OTuL//b//pxXTa6+9JszMzMSOHTtESUmJyMjIEMOHDxf33XefuHr1arOxv/766wKASEtLE5WVleL69eti4sSJAoA4dOiQKCoqEpWVlSIyMlIAEOnp6ZoxX3rpJWFubi727t0rbt68KaKjo0WPHj3E2bNndc75+eefCysrK7Fq1Sq9r8eff/4pAIjQ0NBGX+e1Mcy10VdLP0/Epe+6Ol5f6oQSWYxTp2LIYlylUul87+rVqwUAUVhY2OT7tmzZIgCI3NxcIYQQP/30kwAgPv/88ybHvbfgq6qqEr169RJhYWFa/c6cOSMAaBW7TcVeX/CVl5dr2v773/8KACIzM7PBmLt37xZCCKFSqYRCodCau6qqSpibm4vFixe36HzpQ99inNemY64Ni/GWY7HWtfH6UieUyGfGqduqXxJL1xrIZmZmAAC1Wg0A8PDwgKOjI+bPnw+lUonw8HD0799f5zxZWVmoqKjAyJEjtdpHjRoFMzMzpKWltSr++thqamo0bfXHVB9vTk4OqqqqMGTIEE0fCwsLODs7az2G0dnw2hju2nz33XcGG6s7qN9iPTExUeJIqD1cvnwZffv2lToMIi0sxqnbOHToENatW4esrCyUlZVpiqKWsLCwwPHjx/HKK68gNjYWq1atwpw5c7Bt2zbN1t33KikpAQDNM8p3s7W1RXl5eYvj0FdlZSUA4NVXX8Wrr76q9ZqLi0u7zdtSvDbtd23i4+MRHx9vsPG6i7lz50odArWTWbNmSR0CkRZ+gZO6hT/++APTp0+Hs7Mz0tLSUFpairi4uFaNNXjwYBw8eBAFBQVYunQp9uzZg/Xr1zfZ39bWFgAaLexKSkra9S6Ng4MDAODtt9+GEELrp7PcMeW1ad9rs2fPngbj86fpn1mzZmHWrFmSx8Gf9ru+RJ0Ni3HqFjIzM6FWq7F48WJ4eHhALpdDJpO1eJyCggL8/PPPAO4UU2vWrMHw4cM1bY0ZMmQIevXqhe+//16rPS0tDdXV1RgxYkSL49BX/Uoe6enp7TZHW/HadN5rQ0RE7Y/FOBk1e3t7FBQUID8/H+Xl5U0+3uDm5gYAOHr0KG7duoULFy606nnggoICLFy4ENnZ2aiursb58+fx+++/w9/fv8n3yOVyREVF4bPPPsMnn3yCsrIyZGZmYtGiRXBxcWnXNbjlcjmefvpp7Nq1C1u3bkVZWRlqa2tx+fJlzW6ZTUlJSTHo0oZN4bVp+bUhIqIuRBB1Imjh6g/nzp0T999/v7CwsBBjxowRL774orCwsBAARL9+/cSOHTs0fZcuXSrs7e2Fra2tmD17tti8ebMAIDw9PcUrr7wiFAqFACAGDBgg8vLyxAcffCCsra0FAHH//feLX3/9VeTn54vAwEBhZ2cnevbsKfr06SOWL18uampqRH5+vhg2bJgAIExMTMTw4cPF3r17hRBC1NXViXXr1okBAwYIU1NTYWdnJ6ZPny5ycnI08cXFxTUae3x8vCa2/v37i5MnT4q1a9cKGxsbAUA4OTmJnTt3it27dwsnJycBQNjZ2Yldu3YJIYS4ffu2WLp0qXBzcxMmJibCwcFBzJw5U2RlZTU5pxBCHD58WFhZWYk333yz2etQVlYmxo4dK+zt7QUA0aNHD+Hl5SViY2ObPT5em5ZfG3219PNEXG2jq+P1pU4oUSaEEB1Y+xPpxO27iQyHn6eW43bpXRuvL3VCSXxMhYiIiIhIIizGiYiIiIgkwmKciIionRw9ehTLli1DXV0dpk+fDjc3N8jlcri6uiI0NBQZGRktGs9Q49T79NNPMWrUKFhZWeH+++/H008/jatXrzbod+rUKTz88MNQKBRwcXHB0qVLcfv27Vb1+8tf/gKZTNboT/2a/wcOHEBcXJzOjb+IugoW40RERO3g9ddfx6ZNmxAdHY26ujqcPHkSn376KYqLi3Hq1CmoVCqMHTsWBQUFeo9pqHGAO2vQP/XUU5g9ezYuX76M5ORknDhxApMmTdLaPTYrKwuPPfYYgoODUVRUhM8++wwfffQRFi1apDWevv10GTNmDABg6tSpkMvlCA4O1mzORdRVsRgnIqJ2oVKpEBgYaPRztMbatWuxe/duJCYmwsrKCgAQEBCAMWPGQKFQwN3dHbGxsSgtLcXHH3/corENNc7777+PPn364F//+hdsbGzw4IMP4sUXX0R6errW8qJvvPEGnJ2dsXLlSlhaWiIgIABLly7Fxx9/jOzs7Bb3k8vlKCsra7AhT0REBF5++WVNP6VSiQceeACTJ0/W+uWAqKthMU5ERO0iISEBhYWFRj9HS+Xm5mLFihVYuXIl5HI5AMDExAQHDx7U6ufh4QEAyMvL03tsQ40DAJcuXYKLi4vWJlv9+vUDAPz+++8AgJqaGhw6dAjjxo3T6jdp0iQIIZCcnNyifgBw5MgRzS8od8fy008/4ZFHHtFqj4mJQXp6OuLj41t0bETGhMU4EREBAIQQ2LBhA3x8fGBubg47OztMmzZN665mZGQkzMzM4OzsrGl7/vnnYWlpCZlMhuvXrwMAlixZgqioKOTl5UEmk8HLywubNm2CXC6Ho6MjFi5cCBcXF8jlcgQGBmrdiW3LHMCdYq8jNqxqyqZNmyCEwNSpU3X2U6lUAABra+s2zdfacTw8PBr8IlP/vHh9gf/bb7+hoqJCszlXPU9PTwDQPKuub7+mrF27FkqlskG7nZ0dxo0bh/j4eHAlZuqqWIwTERGAO3chly1bhuXLl6OwsBAnTpzApUuXEBQUhGvXrgG4U2jeu275li1bsHLlSq22+Ph4hISEwNPTE0II5ObmIjIyEuHh4aiqqoJSqUR+fj7OnTuHmpoaTJgwAZcuXWrzHAA0X/qrq6sz3MlpgUOHDsHb2xsKhUJnvzNnzgD4v+ekW6u140RHR+Pq1at45513UF5ejqysLMTHx+Pxxx/X7FxbX5zfeydbLpfDwsJC8+9C336NuXLlClJTUzFz5sxGXx82bBiuXLmCH3/8sUXHR2QsWIwTERFUKhU2bNiAGTNmYP78+bCxsYGfnx/ee+89XL9+HR988IHB5jIxMdHcfff19cXWrVtRXl6Obdu2GWT8KVOmoKysDCtWrDDIeC1RWVmJixcvau4IN+batWvYvXs3lEolAgICmr2D3l7jjBs3DkuXLkVkZCSsra0xZMgQlJeX4z//+Y+mT/1KKD179mzwflNTU81deX37NWbt2rX45z//iR49Gi9JBgwYAADIzMzU88iIjAuLcSIiQlZWFioqKjBy5Eit9lGjRsHMzEzrMRJDGzlyJBQKhdbjMMaqsLAQQgidd8UDAgKgVCoxbdo0pKSkwNTUtFVztXWc5cuX44MPPsCxY8dQUVGB3377DYGBgQgICND8laL+mffGvkBZXV0NCwuLFvW7V0FBAQ4cOIDw8PAm46w/l7rurhMZMxbjRESkWT6ufp3nu9na2qK8vLxd5zc3N0dRUVG7ztERbt26BeDO8TTF0dERx48fxzvvvAMbG5tWz9WWcf7880/ExcXh73//Ox555BFYWlrC3d0dH374IQoKCrBu3ToA0Dy3X1ZWpvX+qqoq3Lp1Cy4uLi3qd6+4uDg899xzmmK+MfWFfP25JepqTKQOgIiIpGdrawsAjRbdJSUl6Nu3b7vNrVar232OjlJfOOrarMbBwUFzvtuiLeNcuHABtbW16NOnj1a7tbU17O3tkZWVBQBwd3eHlZWVZnWVevXP5w8dOrRF/e529epVfPrpp8jJydEZa3V1NQA0eXedyNixGCciIgwZMgS9evXC999/r9WelpaG6upqjBgxQtNmYmICtVptsLlTU1MhhNB8abA95ugojo6OkMlkKC0tbbLPvUsTtlZbxqn/xefPP//Uai8vL0dxcbFmiUMTExNMnjwZJ06cQF1dnea57pSUFMhkMs1z6vr2u1tcXBzmz58Pe3t7nbHWn0snJ6dWHy9RZ8bHVIiICHK5HFFRUfjss8/wySefoKysDJmZmVi0aBFcXFwQERGh6evl5YXi4mLs378farUaRUVFDe6IAoC9vT0KCgqQn5+P8vJyTXFdV1eHmzdvoqamBhkZGViyZAnc3Ny0nhtuyxwpKSmSLW2oUCjg4eGBy5cvN/p6bm4unJycMHfu3AavhYWFwcnJCefOnWt2nraO4+7ujvHjx+PDDz/EiRMnoFKpcOnSJc11fvbZZzV9V6xYgWvXruH1119HZWUlvvvuO6xbtw7h4eHw9vZucT/gzvPfH330EV544YVmj7X+XPr5+TXbl8gYsRgnIiIAd7ZvX716NVatWoX77rsP48aNQ//+/ZGamgpLS0tNv8WLF2P8+PGYN28evL298cYbb2geIbj7y3+LFi2Co6MjfH19MXnyZBQXFwO48+yvn58fLCwsEBQUhIEDB+Krr77Ses66rXNIacqUKcjKymp0BRFda2VXV1ejsLBQa4OcprR1HJlMhqSkJISFheHZZ5+FnZ0dfH198ccff2Dfvn0ICgrS9B08eDC++OILfPnll+jduzdmzpyJZ555Bu+++67WmPr2A4C33noLU6dObbAueWPOnj0LV1fXRh91IeoKZIKr6FMnIpPJsGfPngZrDBNRy3XGz9PChQuRlJSEGzduSB1Ko2bPng0ASEpKavUYubm58PHxwbZt2zB//ny931dXV4e//OUvCA8PxzPPPNPq+Q01Tmdw48YN9O3bF2+++SaioqLaPJ4hri+RgSXxzjgREXUoXV9u7Aq8vLywatUqrFq1ChUVFXq9p7a2Fvv370d5eTnCwsJaPbehxuksYmJi8OCDDyIyMlLqUIjaDYtxIiIiA1u2bBlmz56NsLAwnV/mrJeamop9+/YhJSWl2Z07O2KczmDDhg1IT0/H4cOHW70WO5ExYDFOREQdIjo6Gtu2bUNpaSnc3d2xd+9eqUNqV7GxsYiMjMSaNWua7RscHIydO3dq1utu7KJsngAAHxlJREFULUONI7Xk5GTcvn0bqampsLOzkzoconbFpQ2JiKhDrF69GqtXr5Y6jA712GOP4bHHHpM6DKMTGhqK0NBQqcMg6hC8M05EREREJBEW40REREREEmExTkREREQkERbjREREREQS4aY/1KnIZDL4+/ujb9++UodCZPT27t3Lz1MLnT59GgDg7+8vcSTUHk6fPg1/f39u+kOdSRKLcepU6ndHI6LmXb16FefPn8ekSZOkDoXIaAQEBODFF1+UOgyieizGiYiMVWJiIubOnQumcSIio5XEZ8aJiIiIiCTCYpyIiIiISCIsxomIiIiIJMJinIiIiIhIIizGiYiIiIgkwmKciIiIiEgiLMaJiIiIiCTCYpyIiIiISCIsxomIiIiIJMJinIiIiIhIIizGiYiIiIgkwmKciIiIiEgiLMaJiIiIiCTCYpyIiIiISCIsxomIiIiIJMJinIiIiIhIIizGiYiIiIgkwmKciIiIiEgiLMaJiIiIiCTCYpyIiIiISCIsxomIiIiIJMJinIiIiIhIIizGiYiIiIgkwmKciIiIiEgiLMaJiIiIiCTCYpyIiIiISCIsxomIiIiIJMJinIiIiIhIIizGiYiIiIgkwmKciIiIiEgiLMaJiIiIiCTCYpyIiIiISCImUgdARETNU6vVqKio0GqrrKwEANy8eVOrXSaTwdbWtsNiIyKi1mMxTkRkBIqLi+Hq6ora2toGr9nb22v99/jx43H8+PGOCo2IiNqAj6kQERkBJycnjB07Fj166E7bMpkM8+bN66CoiIiorViMExEZib/+9a/N9unZsydmzJjRAdEQEZEhsBgnIjISM2fOhIlJ008X9uzZExMnTkTv3r07MCoiImoLFuNEREbC2toakyZNarIgF0Jg/vz5HRwVERG1BYtxIiIjMn/+/Ea/xAkAZmZmeOKJJzo4IiIiagsW40RERuSJJ56AQqFo0G5qaorp06fD0tJSgqiIiKi1WIwTERkRuVyOGTNmwNTUVKtdrVbjqaeekigqIiJqLRbjRERG5sknn4RardZqs7a2xoQJEySKiIiIWovFOBGRkXn00Ue1NvoxNTXFvHnzYGZmJmFURETUGizGiYiMjImJCebNm6d5VEWtVuPJJ5+UOCoiImoNFuNEREZo3rx5mkdVnJycMGbMGIkjIiKi1mAxTkRkhAIDA+Hq6goAWLBgAXr0YDonIjJGTW/lRkRtlpiYKHUI1IWNGjUKV65cQe/evflvjdpNv379EBAQIHUYRF2WTAghpA6CqKuSyWRSh0BE1CazZs1CUlKS1GEQdVVJvDNO1M727NmDOXPmSB0GGcjs2bMBoNMUJ3v37sWsWbOkDkOnxMREzJ07F7z3Y3zq/73T/2/v3oOqLvMHjr+PcTkc7pQgXjAuyUJSVjgrJKGyUeoG3gha3Yl0vGAT6LLFosOA1yCKWDVttbRmkkTSBRvFrrLoWpZjXqLVEY1EkYuScFCunu/vD3/nrCcQuR9wP68Z/3n4fD/P8/3CyOc8PN/nEaL3yCJDIYQYwPp7IS6EEKJ9UowLIYQQQghhIlKMCyGEEEIIYSJSjAshhBBCCGEiUowLIYQQQghhIlKMCyGEEEIIYSJSjAshhAns27cPe3t7Pv30U1MPpd/78ssvSUxMRKfTMX36dNzc3FCr1QwbNozw8HBOnjzZqXw9lUcvKyuLsWPHYmtry8iRI3nppZcoLy9vFXfo0CGefPJJNBoNrq6uJCQk0NjY2KW4CRMmoFKp2vxnY2MDwJ49e0hLS+PmzZtdui8hRN+QYlwIIUxA9tzumOTkZNatW8eyZcvQ6XQcPHiQrKwsqqurOXToEPX19Tz11FOUlZV1OGdP5YFb5wjMnj2biIgILl68SF5eHoWFhUyePJmWlhZDXFFREaGhoYSEhFBVVcXu3bvZunUrMTExRvk6Gtee8ePHAxAWFoZarSYkJIRr16516r6EEH1IEUL0GkDJzs429TBED5o1a5Yya9YsUw+jR924cUMJCAjotfzZ2dlKV37dvP7668qoUaOU+vp6RVEUpbm5WfnjH/9oFPPdd98pgLJmzZoO5+2pPIqiKBMnTlSGDh2q6HQ6Q9uGDRsUQDl06JChLTIyUnF3dzeKS09PV1QqlfKf//yn03HPPPOMUltb22o8CxcuVL766iujttjYWCUgIEBpbm7u1L0pyr358y5EP7NTZsaFEOJ/3Pvvv09lZaWph2GkuLiYpKQkVqxYgVqtBsDMzKzVsh4PDw8Azp071+HcPZUHoLS0FFdXV1QqlaFtxIgRAPzyyy8AtLS0sHfvXoKDg43iJk+ejKIo5OXldSoOYP/+/dja2rYay48//sikSZOM2lNSUjh+/DiZmZmdujchRN+QYlwIIfrYoUOHcHNzQ6VSsWHDBgA2btyItbU1Go2GvLw8Jk+ejJ2dHcOHD+fjjz82XLtu3TrUajXOzs4sWrQIV1dX1Go1gYGBHDlyxBAXGxuLhYUFQ4YMMbS9/PLLWFtbo1KpuHLlCgBLliwhPj6ec+fOoVKp8PLyAm4Ve3Z2dqxZs6YvHkkr69atQ1EUwsLC2o2rr68HwM7Orlv9dTWPh4dHqw8y+vXi+gL//Pnz1NXV4ebmZhTn6ekJYFir3tG4O0lNTSUuLq5Vu6OjI8HBwWRmZsryKCH6ISnGhRCij40fP57Dhw8btS1evJilS5dSX1+Pra0t2dnZnDt3Dg8PD+bPn09zczNwq8iOjo7mxo0bxMXFUVJSwrFjx2hpaeHpp5+mtLQUuFXMPv/880Z9vPPOO6xYscKoLTMzk+eeew5PT08URaG4uBjA8NKfTqfrlWdwN3v37sXb2xuNRtNu3HfffQf8d510V3U1z7JlyygvL2f9+vVotVqKiorIzMzkmWeeYdy4ccB/i/PfzmSr1WqsrKyoqKjoVFxbLl26REFBATNnzmzz64899hiXLl3ixIkTnbo/IUTvk2JcCCH6mcDAQOzs7Bg8eDBRUVFcv36dCxcuGMWYmZnh4+ODpaUlvr6+bNy4Ea1Wy7Zt23pkDFOnTqW2tpakpKQeydcZ169f5+effzbMCLeloqKCHTt2EBcXR0BAwF1n0HsrT3BwMAkJCcTGxmJnZ8fo0aPRarW89957hhj9Tij33Xdfq+vNzc0Ns/IdjWtLamoqr7zyCoMGtf1r/aGHHgLg1KlTHbwzIURfkWJcCCH6MQsLCwDDzPid+Pv7o9FoOH36dF8Mq1dVVlaiKEq7s+IBAQHExcUxbdo08vPzMTc371Jf3c2zfPlyNm/ezFdffUVdXR3nz58nMDCQgIAAw18p9Gveb99dRa+pqQkrK6tOxf1WWVkZe/bsITo6+o7j1D/L9mbXhRCmIcW4EELcIywtLamqqjL1MLqtoaEBuHU/d+Ls7MzXX3/N+vXrsbe373Jf3clz+fJl0tLSWLBgAZMmTcLa2hp3d3e2bNlCWVkZ6enpAIZ1+7W1tUbX37hxg4aGBlxdXTsV91tpaWnMnz/fUMy3RV/I65+tEKL/MDP1AIQQQnRfc3Mz165dY/jw4aYeSrfpC8f2DqsZPHgwDg4O3e6rO3nOnj3LzZs3GTp0qFG7nZ0dTk5OFBUVAeDu7o6tra1hdxU9/fr8Rx55pFNxtysvLycrK4szZ860O9ampiaAO86uCyFMR4pxIYS4BxQUFKAoiuGlQbi1rvxuy1v6I2dnZ1QqFTU1NXeM6amTS7uTR//B5/Lly0btWq2W6upqwxaHZmZmTJkyhcLCQnQ6nWFdd35+PiqVyrBOvaNxt0tLS2POnDk4OTm1O1b9s3Rxceny/QoheocsUxFCiAFIp9Px66+/0tLSwsmTJ1myZAlubm5G64a9vLyorq4mNzeX5uZmqqqqWs26Ajg5OVFWVkZJSQlarZbm5mby8/NNtrWhRqPBw8ODixcvtvn14uJiXFxciIyMbPW1qKgoXFxcOHbs2F376W4ed3d3Jk6cyJYtWygsLKS+vp7S0lIWLlwIwLx58wyxSUlJVFRUkJyczPXr1/nmm29IT08nOjoab2/vTsfBrfXfW7duZenSpXe9V/2z9PPzu2usEKJvSTEuhBB9bMOGDYwdOxaAhIQEwsPD2bhxI2+//TZwaznC+fPn2bJlC/Hx8QA8++yznD171pCjoaEBPz8/rKysCAoKYtSoURw4cMBonfXixYuZOHEiL7zwAt7e3qxatcqwTOH2FwxjYmJwdnbG19eXKVOmUF1d3SfPoT1Tp06lqKiozR1E2tsru6mpicrKSqMDcu6ku3lUKhU5OTlERUUxb948HB0d8fX15cKFC+zatYugoCBD7MMPP8xnn33G559/zv3338/MmTOZO3cumzZtMsrZ0TiAN954g7CwsFb7krfl+++/Z9iwYW0udRFCmJZKkRMAhOg1KpWK7OzsVvs9i4ErIiICgJycHJONYdGiReTk5HD16lWTjaEzdu7cSWRkZKcOnCkuLsbHx4dt27YxZ86cDl+n0+mYMGEC0dHRzJ07tyvD7dE8/cHVq1cZPnw4q1evNny466j+8PMuxD0uR2bGhRBiAGrv5cZ7gZeXFytXrmTlypXU1dV16JqbN2+Sm5uLVqslKiqqy333VJ7+IiUlhTFjxhAbG2vqoQgh2iDFuBD9xJtvvml4ce3dd9/t8/5PnDhBVFQU7u7uWFpa8sADD/Doo4+yevXqLudsbGwkLi6OIUOGoNFo2L9/fw+OWNzrEhMTiYiIICoqqt2XOfUKCgrYtWsX+fn5dz25sy/y9AcZGRkcP36cffv2dXkvdiFE75JiXIh+4q9//WurI9L7yqlTpwgMDGTIkCEcOHCAmpoaDh8+zLPPPktBQUGX87711lvs37+f06dPk5mZ2eEZTnFny5YtY9u2bdTU1ODu7s4nn3xi6iH1qjVr1hAbG8vrr79+19iQkBC2b99u2K+7q3oqj6nl5eXR2NhIQUEBjo6Oph6OEOIOZGtDIQRvvvkmDg4OZGZmGtpGjRrFqlWrmDlzZpfz5ubm4u/vj4ODAwsWLACgvr6ekJAQk33wGOjWrl3L2rVrTT2MPhUaGkpoaKiphzHghIeHEx4ebuphCCHuQmbGhRBcvXqVmpqaVrtoWFhYdGsf5osXL7b60/j7779PZWVll3MKIYQQ9xIpxoXoxzIzM7G2tmbQoEE88cQTuLi4YG5ujrW1NY8//jhBQUGMGDECtVqNg4MDr732mtH1+/fv79Be0WPHjuX69etMmjSJf//73+3GKopCRkYGPj4+WFpa4ujoyLRp0zh9+rQh5osvvsDLy4vLly/z4YcfolKpsLGxYcmSJcTHx3Pu3DlUKhVeXl7dvseDBw/i6+uLvb09arUaPz8/PvvsMwA++OADbGxsUKlUODo6kpuby9GjRxk5ciT33Xcff/rTnzrz7RBCCCF6nBTjQvRjS5Ys4dVXX0VRFDZt2sTPP/9MeXk5Tz31FD/88AOJiYn88MMPVFdX8+KLL5Kens6JEycM1+t33NDpdO3289prr+Hv78+JEycYP348Dz/8MG+88Uab+02npKSQmJjI8uXLqayspLCwkNLSUoKCgqioqADg6aefNhyo8uKLL6IoCnV1dWRmZvLcc8/h6emJoigUFxd3+x4rKiqIjIykpKSEsrIybGxsmD17NgDR0dF8//33aDQawsPDmTZtGv7+/syePZvNmzeTlZXV7e+REEII0R1SjAsxQPj6+qLRaLj//vt54YUXAHBzc+OBBx5Ao9EY9mK+fYZ66tSp1NbWkpSU1G5uKysrDh8+zN///nd+97vf8dNPP5GQkICPjw//+te/DHH19fVkZGQwY8YM5syZg729PX5+frz77rtcuXKFzZs39/k9zpo1i+TkZBwdHXFyciIsLIyrV69SVVUFgI+PD2+//TYffvgh27dv5+OPP6axsdHodEQhhBDCVOQFTiEGIAsLCwBaWloMbfq12c3NzV3KaW5uTmxsLLGxsRw5coTU1FRyc3OJiIjgzJkzODo6UlRURF1dHf7+/kbXjh07FgsLC44cOdLFO2qtq/eoj7l9H+4FCxbwxRdfsGjRIv7whz90eweSb7/91nAYirg7/VHs8swGnm+//ZZx48aZehhC3NNkZlwI0crvf/97/vnPfxITE0NVVRUHDhwA4Nq1awDY2Ni0usbBwQGtVtun4wTYu3cvEyZMYPDgwVhaWrZaU663Zs0a6urq5OVRIYQQ/YrMjAshmDlzJtnZ2ZiZGf+X8Oc//5lNmzZx48YN4FbBDbRZdF+7do3hw4f3/mBvc+HCBaZPn86MGTPYunUrQ4cOZf369a0K8ubmZuLi4sjIyCA+Pp7Vq1eTnJzc5X7HjRsnx4N3ws6dO4mMjJRnNgDJXzOE6H1SjAshaGxs5KeffuKRRx4xaj9z5gyAoX306NHY2Nhw9OhRo7gjR47Q1NTEE0880TcD/n+nTp2iubmZxYsX4+HhAYBKpWoV98orrzB//nxmzJjBpUuXWLVqFaGhoQQEBPTpeIUQQojfkmUqQtzD8vPzO7S1IcD06dPZuXMn165do6amhry8PP72t78RHh5uKMbVajXx8fHs3r2bjz76iNraWk6dOkVMTAyurq4sXLjwrv04OTlRVlZGSUkJWq22y2vc4dbLnQBffvklDQ0NnD17ttW69XfeeYdhw4YxY8YM4NahOb6+vsyePZva2tou9y2EEEL0CEUI0WsAJTs7u0Oxb731luLi4qIAirW1tTJjxgwlMzNT0Wg0CqA8+OCDysGDB5XU1FTF3t5eARQXFxdl+/btyo4dOwzXOjo6Kh9//LGiKIqyb98+xdbWVlm9enW7fX/++edKZGSk4unpqVhaWioWFhaKt7e3kpKSojQ0NBjF6nQ6JT09XXnooYcUc3NzxdHRUZk+fbpy5swZQ0xJSYny2GOPKYBiZmamPP7448onn3yiKIqiHDt2TBk5cqRiZWWljB8/XklMTOzWPSYkJChOTk6Kg4ODEhERoWzYsEEBFE9PT2XMmDGKSqVSnJyclMOHDyuKoihLly5VBg0apACKvb29cvTo0Y59M//frFmzlFmzZnXqmv912dnZivy6GZjk512IXrdTpSiKYpqPAULc+1QqFdnZ2Tz//POmHoroIfo1tLL+ueP0a8bl183AIz/vQvS6HFmmIoQQQgghhIlIMS6EEKJf+/LLL0lMTESn0zF9+nTc3NxQq9UMGzaM8PBwTp482al8PZVHLysri7Fjx2Jra8vIkSN56aWXKC8vbxV36NAhnnzySTQaDa6uriQkJNDY2NiluAkTJqBSqdr8p996dM+ePaSlpRntuS+E6H+kGBdCCNFvJScns27dOpYtW4ZOp+PgwYNkZWVRXV3NoUOHqK+v56mnnqKsrKzDOXsqD0B2djazZ88mIiKCixcvkpeXR2FhIZMnTzY6sKqoqIjQ0FBCQkKoqqpi9+7dbN26lZiYGKN8HY1rz/jx4wEICwtDrVYTEhJiOCNACNH/SDEuhBADTH19PYGBgQO+j7tJTU1lx44d7Ny5E1tbWwACAgIYP348Go0Gd3d31qxZQ01NDR988EGncvdUnn/84x8MHTqUV199FXt7e8aMGcNf/vIXjh8/brSzz6pVqxgyZAgrVqzA2tqagIAAEhIS+OCDDzh9+nSn49RqNbW1tSiKYvRv4cKFRvvsx8XF8eijjzJlyhSjDwdCiP5DinEhhBhg3n///V4/SbQv+mhPcXExSUlJrFixArVaDYCZmRmffvqpUZx+f/lz5851OHdP5QEoLS3F1dXVaH/7ESNGAPDLL78A0NLSwt69ewkODjaKmzx5MoqikJeX16k4gP379xs+oNw+lh9//JFJkyYZtaekpHD8+HEyMzM7dW9CiL4hxbgQQvQyRVHIyMjAx8cHS0tLHB0dmTZtmtFMZ2xsLBYWFgwZMsTQ9vLLL2NtbY1KpeLKlSsALFmyhPj4eM6dO4dKpcLLy4t169ahVqtxdnZm0aJFuLq6olarCQwMNJqd7U4fcKsA7Oi+9d21bt06FEUhLCys3bj6+noA7OzsutVfV/N4eHi0+tCiXy+uL/DPnz9PXV2dYV98PU9PTwDDWvWOxt1JamoqcXFxrdodHR0JDg4mMzNTdrQRoh+SYlwIIXpZSkoKiYmJLF++nMrKSgoLCyktLSUoKIiKigrgVvH52y0w33nnHVasWGHUlpmZyXPPPYenpyeKolBcXExsbCzR0dHcuHGDuLg4SkpKOHbsGC0tLTz99NOUlpZ2uw/A8CKgTqfruYdzB3v37sXb2xuNRtNu3HfffQf8d510V3U1z7JlyygvL2f9+vVotVqKiorIzMzkmWeeYdy4ccB/i/PfzmSr1WqsrKwMPwMdjWvLpUuXKCgoYObMmW1+/bHHHuPSpUucOHGiU/cnhOh9UowLIUQvqq+vJyMjgxkzZjBnzhzs7e3x8/Pj3Xff5cqVK2zevLnH+jIzMzPMvvv6+rJx40a0Wi3btm3rkfxTp06ltraWpKSkHsl3J9evX+fnn382zAi3paKigh07dhAXF0dAQMBdZ9B7K09wcDAJCQnExsZiZ2fH6NGj0Wq1vPfee4YY/U4o9913X6vrzc3NDbPyHY1rS2pqKq+88gqDBrX9a/2hhx4C4NSpUx28MyFEX5FiXAghelFRURF1dXX4+/sbtY8dOxYLCwujZSQ9zd/fH41GY7QcZiCorKxEUZR2Z8UDAgKIi4tj2rRp5OfnY25u3qW+uptn+fLlbN68ma+++oq6ujrOnz9PYGAgAQEBhr9I6Ne8t/UCZVNTE1ZWVp2K+62ysjL27NlDdHT0Hcepf5btza4LIUxDinEhhOhF+i3l9Hs/387BwQGtVtur/VtaWlJVVdWrffS0hoYG4NbY78TZ2Zmvv/6a9evXY29v3+W+upPn8uXLpKWlsWDBAiZNmoS1tTXu7u5s2bKFsrIy0tPTAQxr9Gtra42uv3HjBg0NDbi6unYq7rfS0tKYP3++oZhvi76Q1z9bIUT/YWbqAQghxL3MwcEBoM2i+9q1awwfPrzX+m5ubu71PnqDvnBs77CawYMHG55td3Qnz9mzZ7l58yZDhw41arezs8PJyYmioiIA3N3dsbW1Neyuoqdfi//II490Ku525eXlZGVlcebMmXbH2tTUBHDH2XUhhOlIMS6EEL1o9OjR2NjYcPToUaP2I0eO0NTUxBNPPGFoMzMzo7m5ucf6LigoQFEUw4uEvdFHb3B2dkalUlFTU3PHmN9uTdhV3cmj/5Bz+fJlo3atVkt1dbVhi0MzMzOmTJlCYWEhOp3OsK47Pz8flUplWKfe0bjbpaWlMWfOHJycnNodq/5Zuri4dPl+hRC9Q5apCCFEL1Kr1cTHx7N7924++ugjamtrOXXqFDExMbi6urJw4UJDrJeXF9XV1eTm5tLc3ExVVVWrWVIAJycnysrKKCkpQavVGoprnU7Hr7/+SktLCydPnmTJkiW4ubkZrSXuTh/5+fl9srWhRqPBw8ODixcvtvn14uJiXFxciIyMbPW1qKgoXFxcOHbs2F376W4ed3d3Jk6cyJYtWygsLKS+vp7S0lLD93TevHmG2KSkJCoqKkhOTub69et88803pKenEx0djbe3d6fj4Nb6761bt7J06dK73qv+Wfr5+d01VgjRt6QYF0KIXpacnMzatWtZuXIlDzzwAMHBwTz44IMUFBRgbW1tiFu8eDETJ07khRdewNvbm1WrVhmWFdz+QmBMTAzOzs74+voyZcoUqqurgVvrgf38/LCysiIoKIhRo0Zx4MABo7XX3e2jr0ydOpWioqI2dxBpb6/spqYmKisrjQ7IuZPu5lGpVOTk5BAVFcW8efNwdHTE19eXCxcusGvXLoKCggyxDz/8MJ999hmff/45999/PzNnzmTu3Lls2rTJKGdH4wDeeOMNwsLCWu1L3pbvv/+eYcOGtbnURQhhWipFTgAQoteoVCqys7Nb7e0sBq6IiAgAcnJyTDwSY4sWLSInJ4erV6+aeiit7Ny5k8jIyE4dOFNcXIyPjw/btm1jzpw5Hb5Op9MxYcIEoqOjmTt3bleG26N5+oOrV68yfPhwVq9eTXx8fKeu7a8/70LcQ3JkZlwIIe4R7b3wONB4eXmxcuVKVq5cSV1dXYeuuXnzJrm5uWi1WqKiorrcd0/l6S9SUlIYM2YMsbGxph6KEKINUowLIYTolxITE4mIiCAqKqrdlzn1CgoK2LVrF/n5+Xc9ubMv8vQHGRkZHD9+nH379nV5L3YhRO+SYlwIIQa4ZcuWsW3bNmpqanB3d+eTTz4x9ZB6zJo1a4iNjeX111+/a2xISAjbt2837NfdVT2Vx9Ty8vJobGykoKAAR0dHUw9HCHEHsrWhEEIMcGvXrmXt2rWmHkavCQ0NJTQ01NTDGHDCw8MJDw839TCEEHchM+NCCCGEEEKYiBTjQgghhBBCmIgU40IIIYQQQpiIFONCCCGEEEKYiBTjQgghhBBCmIicwClEL1KpVKYeghBCdMusWbPkBE4hek+ObG0oRC/Kzs429RCEEKJbRowYYeohCHFPk5lxIYQQQgghTCNH1owLIYQQQghhIlKMCyGEEEIIYSJSjAshhBBCCGEiZoC8Ii2EEEIIIUTf+/b/AByvrIq3yQSdAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<IPython.core.display.Image object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 158
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hjqOTrjvxji7",
        "colab_type": "text"
      },
      "source": [
        "## Loss"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6x6W5AYLaqMw",
        "colab_type": "text"
      },
      "source": [
        "### Tutorial"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nuExhj88Y78P",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0221d601-7ffc-45ca-ab39-b1aa3381b2d7"
      },
      "source": [
        "# vocab 개수\n",
        "n_test_vocab = 4\n",
        "# 문자열 개수\n",
        "n_test_seq = 8\n",
        "\n",
        "# 정답\n",
        "y_true = np.random.randint(0, n_test_vocab, (1, n_test_seq))\n",
        "y_true[:,n_test_seq - 2:] = 0\n",
        "y_true"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[3, 0, 3, 1, 2, 3, 0, 0]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 159
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aBnHCjOLZlrJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8559699f-ffeb-4d95-c54d-5ac0629a72ad"
      },
      "source": [
        "# 예측 값\n",
        "y_pred = np.random.random((1, n_test_seq, n_test_vocab))\n",
        "y_pred = tf.nn.softmax(y_pred, axis=-1)  # 확률 값으로 변경\n",
        "y_pred"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 8, 4), dtype=float64, numpy=\n",
              "array([[[0.25100796, 0.29038113, 0.19352738, 0.26508353],\n",
              "        [0.156816  , 0.34132925, 0.29911483, 0.20273992],\n",
              "        [0.13582323, 0.32769097, 0.29060293, 0.24588287],\n",
              "        [0.17807239, 0.19304151, 0.27926074, 0.34962536],\n",
              "        [0.3500575 , 0.24878816, 0.2201442 , 0.18101014],\n",
              "        [0.13758752, 0.2658076 , 0.34122989, 0.25537499],\n",
              "        [0.29718426, 0.2201506 , 0.28483235, 0.19783279],\n",
              "        [0.25733929, 0.17902035, 0.17726573, 0.38637464]]])>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 160
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qSxBrmylauiQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4efb9709-aace-4d0f-d435-82d2a9bfdb96"
      },
      "source": [
        "# loss 계산\n",
        "loss = tf.keras.losses.SparseCategoricalCrossentropy(reduction=tf.keras.losses.Reduction.NONE)(y_true, y_pred)\n",
        "loss"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 8), dtype=float64, numpy=\n",
              "array([[1.32771027, 1.85268211, 1.40289998, 1.64485002, 1.51347256,\n",
              "        1.3650223 , 1.21340287, 1.35735989]])>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 161
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ne8JKo0Za6Oj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9ba9cebd-9c90-4c6c-dfa5-08ea1a07cebc"
      },
      "source": [
        "# true token이 0인 부분은 0, 아닌 부분은 1인 mask 생성\n",
        "mask = tf.cast(tf.math.not_equal(y_true, 0), dtype=loss.dtype)\n",
        "mask"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 8), dtype=float64, numpy=array([[1., 0., 1., 1., 1., 1., 0., 0.]])>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 162
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GWZUCZhObNF8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ecb215ce-395f-404e-e277-b57f8239cd21"
      },
      "source": [
        "# loss에 mask를 취함. mask가 0인 부분은 모두 0\n",
        "loss *= mask\n",
        "loss"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 8), dtype=float64, numpy=\n",
              "array([[1.32771027, 0.        , 1.40289998, 1.64485002, 1.51347256,\n",
              "        1.3650223 , 0.        , 0.        ]])>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 163
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TOWlO39Vaoit",
        "colab_type": "text"
      },
      "source": [
        "### 함수"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fZwfXjFux190",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def lm_loss(y_true, y_pred):\n",
        "    \"\"\"\n",
        "    loss 계산 함수\n",
        "    :param y_true: 정답 (bs, n_seq)\n",
        "    :param y_pred: 예측 값 (bs, n_seq, n_vocab)\n",
        "    \"\"\"\n",
        "    # loss 계산\n",
        "    loss = tf.keras.losses.SparseCategoricalCrossentropy(reduction=tf.keras.losses.Reduction.NONE)(y_true, y_pred)\n",
        "    # pad(0) 인 부분 mask\n",
        "    mask = tf.cast(tf.math.not_equal(y_true, 0), dtype=loss.dtype)\n",
        "    loss *= mask\n",
        "    return loss"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZV5zGuQQL_mK",
        "colab_type": "text"
      },
      "source": [
        "### Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "larwLi_RbefS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "671dd243-0809-42eb-b2bb-b24dc0ebbaa1"
      },
      "source": [
        "# loss 함수 기능 확인\n",
        "loss2 = lm_loss(y_true, y_pred)\n",
        "assert np.array_equal(loss, loss2)\n",
        "loss2"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 8), dtype=float64, numpy=\n",
              "array([[1.32771027, 0.        , 1.40289998, 1.64485002, 1.51347256,\n",
              "        1.3650223 , 0.        , 0.        ]])>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 165
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fAR-ddNrMGby",
        "colab_type": "text"
      },
      "source": [
        "## Accuracy"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jWIUoq0cbYT8",
        "colab_type": "text"
      },
      "source": [
        "### Tutorial"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h1WlqKVDbbJi",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fa9257c2-985a-407a-99f2-d4f45d9d859f"
      },
      "source": [
        "# vocab 개수\n",
        "n_test_vocab = 3\n",
        "# 문자열 개수\n",
        "n_test_seq = 16\n",
        "\n",
        "# 정답\n",
        "y_true = np.random.randint(0, n_test_vocab, (1, n_test_seq))\n",
        "y_true[:,n_test_seq - 2:] = 0\n",
        "y_true"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0, 2, 1, 0, 2, 0, 2, 2, 2, 2, 1, 0, 0, 1, 0, 0]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 166
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xb_oYi3wbbCz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "45d7ba4f-281e-492e-b5bd-8cc35afd5b33"
      },
      "source": [
        "# 예측 값\n",
        "y_pred = np.random.random((1, n_test_seq, n_test_vocab))\n",
        "y_pred = tf.nn.softmax(y_pred, axis=-1)  # 확률 값으로 변경\n",
        "y_pred"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 16, 3), dtype=float64, numpy=\n",
              "array([[[0.42962916, 0.20050436, 0.36986648],\n",
              "        [0.33637154, 0.31709933, 0.34652913],\n",
              "        [0.31573923, 0.47484686, 0.20941391],\n",
              "        [0.25999641, 0.45710555, 0.28289804],\n",
              "        [0.49895507, 0.26704405, 0.23400088],\n",
              "        [0.2431871 , 0.38622575, 0.37058715],\n",
              "        [0.3297653 , 0.36277189, 0.30746282],\n",
              "        [0.37566364, 0.30902627, 0.31531009],\n",
              "        [0.38077417, 0.33740447, 0.28182136],\n",
              "        [0.34309708, 0.29835482, 0.35854811],\n",
              "        [0.47506772, 0.26487238, 0.2600599 ],\n",
              "        [0.24959169, 0.32607041, 0.42433791],\n",
              "        [0.36358922, 0.32441693, 0.31199385],\n",
              "        [0.5126483 , 0.22688291, 0.26046879],\n",
              "        [0.33714714, 0.1927852 , 0.47006767],\n",
              "        [0.48391873, 0.31784703, 0.19823424]]])>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 167
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7fU1IEMAcTHF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "30249c65-f2cb-46cc-f543-c2f2894b7490"
      },
      "source": [
        "# 예측 값 중 최대인 index번호를 vocab으로 예측\n",
        "y_pred_class = tf.cast(K.argmax(y_pred, axis=-1), tf.float32)\n",
        "y_pred_class"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 16), dtype=float32, numpy=\n",
              "array([[0., 2., 1., 1., 0., 1., 1., 0., 0., 2., 0., 2., 0., 0., 2., 0.]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 168
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tDOlTxMmcmWQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5f1d87d0-dd71-4b02-db6b-26ee57f3bce3"
      },
      "source": [
        "# 두 값이 같으면 정답(1), 다르면 오류(0)\n",
        "matches = tf.cast(K.equal(y_true, y_pred_class), tf.float32)\n",
        "matches"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 16), dtype=float32, numpy=\n",
              "array([[1., 1., 1., 0., 0., 0., 0., 0., 0., 1., 0., 0., 1., 0., 0., 1.]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 169
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tGdUhsjbc2nG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3b03eb18-85a1-4e8b-a471-cfe3a9f5dc31"
      },
      "source": [
        "# true token이 0인 부분은 0, 아닌 부분은 1인 mask 생성\n",
        "mask = tf.cast(tf.math.not_equal(y_true, 0), dtype=matches.dtype)\n",
        "mask"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 16), dtype=float32, numpy=\n",
              "array([[0., 1., 1., 0., 1., 0., 1., 1., 1., 1., 1., 0., 0., 1., 0., 0.]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 170
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K6FbhVs1dH5Z",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "26a2a02b-292c-43d2-cbb8-7c79bae68156"
      },
      "source": [
        "# matches에 mask를 취함. mask가 0인 부분은 모두 0\n",
        "matches *= mask\n",
        "matches"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(1, 16), dtype=float32, numpy=\n",
              "array([[0., 1., 1., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0.]],\n",
              "      dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 171
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4f4Q2iRedUy3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8b01bb93-6cbd-4fe7-c401-fa4f6fb7a1e8"
      },
      "source": [
        "# matches합을 mask합으로 나누면 정확도 / mask가 모두 0일 경우를 대비해서 분모에 K.maximum(K.sum(mask), 1) 사용\n",
        "accuracy = K.sum(matches) / K.maximum(K.sum(mask), 1)\n",
        "accuracy"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(), dtype=float32, numpy=0.33333334>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 172
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qQqF2KJVcLBF",
        "colab_type": "text"
      },
      "source": [
        "### 함수"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "233nfwn6ba17",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def lm_acc(y_true, y_pred):\n",
        "    \"\"\"\n",
        "    acc 계산 함수\n",
        "    :param y_true: 정답 (bs, n_seq)\n",
        "    :param y_pred: 예측 값 (bs, n_seq, n_vocab)\n",
        "    \"\"\"\n",
        "    # 정답 여부 확인\n",
        "    y_pred_class = tf.cast(K.argmax(y_pred, axis=-1), tf.float32)\n",
        "    matches = tf.cast(K.equal(y_true, y_pred_class), tf.float32)\n",
        "    # pad(0) 인 부분 mask\n",
        "    mask = tf.cast(tf.math.not_equal(y_true, 0), dtype=matches.dtype)\n",
        "    matches *= mask\n",
        "    # 정확도 계산\n",
        "    accuracy = K.sum(matches) / K.maximum(K.sum(mask), 1)\n",
        "    return accuracy"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DYYcdK-_MRni",
        "colab_type": "text"
      },
      "source": [
        "### Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JeWKJfTPdjuA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "959d995c-0720-4e9f-9202-2631d61f2989"
      },
      "source": [
        "accuracy2 = lm_acc(y_true, y_pred)\n",
        "assert accuracy == accuracy2\n",
        "accuracy2"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(), dtype=float32, numpy=0.33333334>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 174
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ey3sghogI4rz",
        "colab_type": "text"
      },
      "source": [
        "## Scheduler\n",
        "\n",
        "![](https://paul-hyun.github.io/assets/transformer/learning_rate.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VKkOrvxoJM1h",
        "colab_type": "text"
      },
      "source": [
        "### Tutorial"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0uUiLdINLVw_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "46e0dd74-c434-416a-a0ac-0ae267021a60"
      },
      "source": [
        "warmup_steps = tf.dtypes.cast(1000, tf.float32)\n",
        "step_num = tf.dtypes.cast(1100, tf.float32)\n",
        "d_model = tf.dtypes.cast(config.d_model, tf.float32)\n",
        "\n",
        "warmup_steps, step_num, d_model"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(<tf.Tensor: shape=(), dtype=float32, numpy=1000.0>,\n",
              " <tf.Tensor: shape=(), dtype=float32, numpy=1100.0>,\n",
              " <tf.Tensor: shape=(), dtype=float32, numpy=256.0>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 175
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7yjfmrczLf2a",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9cbb15f3-0637-4442-b754-3233e99e1493"
      },
      "source": [
        "# step_num ** -0.5 (https://www.tensorflow.org/api_docs/python/tf/math/rsqrt)\n",
        "arg1 = tf.math.rsqrt(step_num)\n",
        "arg1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(), dtype=float32, numpy=0.030151134>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 176
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qq1kSzDPNXue",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d1228792-8981-442b-a053-016c875d9ba8"
      },
      "source": [
        "# step_num * (warmup_steps ** -1.5)\n",
        "arg2 = step_num * (warmup_steps ** -1.5)\n",
        "arg2"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(), dtype=float32, numpy=0.03478505>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 177
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o6B3pN97Nubx",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9bdcdc0e-7ad2-4d7d-c5d9-9fe065dfd2e5"
      },
      "source": [
        "# 두 값중 작은 값 선택\n",
        "arg = tf.math.minimum(arg1, arg2)\n",
        "arg"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(), dtype=float32, numpy=0.030151134>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 178
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N1A4vH27OPFN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fd9a84ce-b35d-4ee1-f245-c2cc79d6037c"
      },
      "source": [
        "# (d_model ** -0.5) * arg\n",
        "lr = tf.math.rsqrt(d_model) * arg\n",
        "lr"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(), dtype=float32, numpy=0.0018844459>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 179
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A7k-JyedJK6J",
        "colab_type": "text"
      },
      "source": [
        "### Class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VPWnDRRMaP4o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class InverseSquareRootSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
        "    \"\"\"\n",
        "    TransformerSchedule class\n",
        "    \"\"\"\n",
        "    def __init__(self, d_model, warmup_steps=4000):\n",
        "        \"\"\"\n",
        "        생성자\n",
        "        :param d_model: 모델 hidden\n",
        "        :param warmup_steps: warmup steps\n",
        "        \"\"\"\n",
        "        super().__init__()\n",
        "\n",
        "        self.d_model = tf.cast(d_model, tf.float32)\n",
        "        self.warmup_steps = tf.cast(warmup_steps, tf.float32)\n",
        "\n",
        "    def __call__(self, step_num):\n",
        "        \"\"\"\n",
        "        learning rate 계산\n",
        "        :param step_num: 현재 step number\n",
        "        :retrun: 계산된 learning rate\n",
        "        \"\"\"\n",
        "        arg1 = tf.math.rsqrt(step_num)\n",
        "        arg2 = step_num * (self.warmup_steps**-1.5)\n",
        "        arg = tf.math.minimum(arg1, arg2)\n",
        "        lr = tf.math.rsqrt(self.d_model) * arg\n",
        "        return lr"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HNHVGEgyacKg",
        "colab_type": "text"
      },
      "source": [
        "### Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WG8ZT6nPadip",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "88181728-f339-43e3-acab-6670c4240fa1"
      },
      "source": [
        "# compute lr\n",
        "test_schedule = InverseSquareRootSchedule(config.d_model, warmup_steps=100)\n",
        "lrs = []\n",
        "for step in range(1000):\n",
        "    lrs.append(test_schedule(float(step)).numpy())\n",
        "\n",
        "# draw\n",
        "plt.plot(lrs, 'r-', label='learning_rate')\n",
        "plt.xlabel('Step')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEGCAYAAABsLkJ6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5RU1Zn38e8DhFaUNNcgN+1GWp1mjIod1DiaqFExJrDWhETwNdGEDJmJGi8ZoyRZZmIuxneyZJKoiSTES1QQifoyJoE4QiYSFWzUGIEubG4CXmgVEEUhwPP+sU/ZRVHdXd1dXaerzu+zVq2uOpfqfSitX5+z93m2uTsiIpI8PeJugIiIxEMBICKSUAoAEZGEUgCIiCSUAkBEJKF6xd2A9hg0aJBXVVXF3QwRkZKxfPny1919cK51JRUAVVVV1NfXx90MEZGSYWYbWlqnS0AiIgmlABARSSgFgIhIQikAREQSSgEgIpJQCgARkYRSAIiIJJQCoCtt3gwPPBB3K0REclIAdKWf/hQ+9zlYuzbuloiIHEAB0JVWrQo/f/vbeNshIpKDAqArpVLhpwJARLohBUBX2b0b1qyBykpYuhQ2bYq7RSIi+1EAdJW1a2HvXrj88vD6wQfjbY+ISJa8AsDMxptZyswazey6HOsrzOz+aP1SM6vKWDc9Wp4ys3Mzlvczs3lm1mBmq8zslEIcULfR0BB+TpgAY8boMpCIdDttBoCZ9QRuBc4DaoEpZlabtdlUYKu7jwZmADdF+9YCk4ExwHjgtuj9AH4CLHD3Y4DjgFWdP5xuJH39/+ij4TOfgccfh1dfjbdNIiIZ8jkDGAc0uvtad98NzAEmZm0zEbgrej4POMvMLFo+x913ufs6oBEYZ2aVwOnALAB33+3u2zp/ON1IQwMMHQof/CBccAG4w/33x90qEZH35RMAw4GNGa83RctybuPue4DtwMBW9q0GmoA7zOxZM/uVmR2S65eb2TQzqzez+qampjya202kUuGvf4DaWhg7Fu65J942iYhkiKsTuBcwFvi5u58AvAMc0LcA4O4z3b3O3esGD845q1n34x7OAI45pnnZRRdBfX1z34CISMzyCYDNwMiM1yOiZTm3MbNeQCXwRiv7bgI2ufvSaPk8QiCUh9dfh61bm88AACZPhh49dBYgIt1GPgHwNFBjZtVm1pvQqTs/a5v5wMXR80nAInf3aPnkaJRQNVADLHP3V4GNZpb+hjwLWNnJY+k+0h3AmWcAQ4fC2WfDvffCvn3xtEtEJEObARBd078MWEgYqTPX3VeY2Q1mNiHabBYw0MwagauJLue4+wpgLuHLfQFwqbvvjfa5HLjXzJ4Hjgd+WLjDiln6Mk/mGQCEy0Dr18MTTxS9SSIi2Sz8oV4a6urqvL6+Pu5mtO2aa+CWW+Dtt6Fnz+blb78NQ4bAhRfCL38ZX/tEJDHMbLm71+VapzuBu0JDA9TU7P/lD3DooaEvYPZs2LEjnraJiEQUAF0hldr/+n+madPgnXfgvvuK2yYRkSwKgELbvTvUAcq+/p82bhx8+MMwc2Zx2yUikkUBUGhr1oQicC0FgFk4C3jmGVi+vLhtExHJoAAotPQIoJYuAUEYDXTwwXD77cVpk4hIDgqAQkvfA3DUUS1vU1kZOoPvuw+2by9Ou0REsigACq2hAYYNC0XgWnPZZaEzeNas4rRLRCSLAqDQMovAtWbsWDj99DBx/J49Xd8uEZEsCoBCcm99CGi2q66CDRvg4Ye7tl0iIjkoAAqpqenAInCt+fSnYdQomDGja9slIpKDAqCQchWBa03PnvC1r4XaQEuXtr29iEgBKQAKqaUicK350pegXz+46aauaZOISAsUAIWUSsFBB8Hhh+e/T9++4SzgoYfghRe6rm0iIlkUAIXU0BDG//do5z/r174WCsX9sHwqYotI96cAKKR8h4BmGzgQvvrVMGn86tWFb5eISA4KgELZtSsUgcu3Azjb178OFRVw442FbZeISAsUAIWyZk2Y6rEjZwAAH/pQKBL3m9+EIBER6WIKgELJpwhcW77xDejdG66/vjBtEhFphQKgUPIpAteWYcPgiivCxPHPPVeYdomItEABUCgNDTB8eBjW2RnXXgv9+8P06YVpl4hICxQAhdLREUDZ+vULX/4LFsCf/tT59xMRaYECoBDaWwSuLZddBiNGhLOBffsK854iIlkUAIWwZQts21aYMwAIs4V973uwbFnoDxAR6QJ5BYCZjTezlJk1mtl1OdZXmNn90fqlZlaVsW56tDxlZudmLF9vZn8zs+fMrL4QBxOb9haBy8cXvhAmkP/GN2DHjsK9r4hIpM0AMLOewK3AeUAtMMXMarM2mwpsdffRwAzgpmjfWmAyMAYYD9wWvV/aGe5+vLvXdfpI4tSRInBt6dEDfvYzePVV+P73C/e+IiKRfM4AxgGN7r7W3XcDc4CJWdtMBO6Kns8DzjIzi5bPcfdd7r4OaIzer7ykUuGyzciRhX3fcePgi18M8wWoRISIFFg+ATAc2JjxelO0LOc27r4H2A4MbGNfB/5oZsvNbFpLv9zMpplZvZnVNzU15dHcGHS0CFw+brwxhMvll4fOZhGRAomzE/if3H0s4dLSpWZ2eq6N3H2mu9e5e93gwYOL28J8FWoIaC5DhoQqoX/8ozqERaSg8gmAzUDmtY0R0bKc25hZL6ASeKO1fd09/XML8BClemlo1y5Yt66wHcDZ/u3f4JRT4Morw7STIiIFkE8APA3UmFm1mfUmdOrOz9pmPnBx9HwSsMjdPVo+ORolVA3UAMvM7BAz6wtgZocA5wClORtKY2PnisDlo0cP+NWv4K23QgiIiBRAmwEQXdO/DFgIrALmuvsKM7vBzCZEm80CBppZI3A1cF207wpgLrASWABc6u57gSHAEjP7K7AM+J27LyjsoRVJIYrA5aO2Fr71LbjvPvj977v2d4lIIpiXUMdiXV2d19d3s1sGfvjD8MW8Y0eY1asr7doFY8fC9u3w/PMwYEDX/j4RKXlmtrylofa6E7izUqlQtqGrv/whTBhz993w2muhX6CEwltEuh8FQGc1NHTt9f9sJ54I3/0uzJ2rUUEi0ikKgM4odBG4fF17LZx6Klx6KWzYUNzfLSJlQwHQGa+9Fq7HF/MMAKBnzzB1pDtcdBHs2VPc3y8iZUEB0BnpInDFDgCA6mr4xS9gyRL45jeL//tFpOQpADqjWENAW3LhhaEz+D//Ex5+OJ42iEjJUgB0RroI3IgR8bVhxgyoq4NLLoE1a+Jrh4iUHAVAZ6RHAHVFEbh8VVTAAw+ENkyaBDt3xtcWESkpCoDO6MoicO1RVRU6hf/611A+WvcHiEgeFAAd9d57XV8Erj3OPx9+9KNwf8ANN8TdGhEpAb3ibkDJamwMf2l3hzOAtGuugZUr4T/+IwTTBRfE3SIR6cZ0BtBRcY8AysUMbr893CR2ySVhUnkRkRYoADoqfQ/AUUfF245sFRXw0EMwdGi4LJRup4hIFgVAR6VSYQ7gQw6JuyUHGjwYFi4MZwTnnAObs+fvERFRAHRcsYvAtVdNDSxYAFu3wvjx4aeISAYFQEeki8B15wCAMHfAww/D6tXw6U/DO+/E3SIR6UYUAB3x6qthesbu1AHckjPPDGWjn3wSPvUp3SgmIu9TAHREnEXgOmLSpHCj2J//HM4EFAIiggKgY7rjENC2XHgh3HUXLF4MEyfCu+/G3SIRiZkCoCNSKejTB4YPj7sl7XPRRXDHHfDYY+FM4O23426RiMRIAdAR3aEIXEddfDHceSf86U/wiU/Am2/G3SIRiUkJfoN1A6UwAqg1X/gC/Pa38NxzcPrp8PLLcbdIRGKQVwCY2XgzS5lZo5ldl2N9hZndH61famZVGeumR8tTZnZu1n49zexZM3ukswdSNO++C+vXl9b1/1wmToQ//CHMKXzqqaG2kYgkSpsBYGY9gVuB84BaYIqZ1WZtNhXY6u6jgRnATdG+tcBkYAwwHrgter+0K4BVnT2IouqOReA66owzQqfwjh3w0Y/CE0/E3SIRKaJ8zgDGAY3uvtbddwNzgIlZ20wE7oqezwPOMjOLls9x913uvg5ojN4PMxsBnA/8qvOHUUSlNgS0LXV14Yu/sjLcMzB7dtwtEpEiyScAhgMbM15vipbl3Mbd9wDbgYFt7PtfwDeAfa39cjObZmb1Zlbf1NSUR3O7WHoIaHcrAtcZRx0FTz0F48aF4aLf+54mlRFJgFg6gc3sU8AWd1/e1rbuPtPd69y9bvDgwUVoXRu6cxG4zhg4EB59NHQQX399+Kl7BUTKWj4BsBkYmfF6RLQs5zZm1guoBN5oZd9TgQlmtp5wSelMM7unA+0vvoaG0u8AbklFRRgi+v3vwz33hM7hdevibpWIdJF8AuBpoMbMqs2sN6FTd37WNvOBi6Pnk4BF7u7R8snRKKFqoAZY5u7T3X2Eu1dF77fI3S8qwPF0rVIpAtcZZvCtb8Ejj4Qv/xNPDFVFRaTstBkA0TX9y4CFhBE7c919hZndYGYTos1mAQPNrBG4Grgu2ncFMBdYCSwALnX3vYU/jCJ55ZUwYqZczwAynX8+1NfD4YfDJz8Z5hne12p3jYiUGPMS6uyrq6vz+vr6+BqweHEYKfPoo+Eu2iTYuRO+8pVwSeiss+Duu2HYsLhbJSJ5MrPl7l6Xa53uBG6PUiwC11l9+oQv/V/+MpSUPu44+O//jrtVIlIACoD2SKXC6J9SKwLXWWbw5S/D8uUwYgRMmACXX65RQiIlTgHQHukicGZxtyQexxwT7he48kq45Rb4yEdCP4GIlCQFQHuU+wigfFRUwIwZoY7Q1q1w8snwzW/Crl1xt0xE2kkBkK933w2F05IeAGnjx8OKFaG89I03hvmHly2Lu1Ui0g4KgHy9+GK4DyBJHcBt6dcPZs0KZwNvvQWnnALXXKOJZkRKhAIgX+VWBK6Qxo+HF16AqVPhxz+G2lp46CHVExLp5hQA+SrHInCFVFkJM2fCkiXhzOCf/zlMO7l2bdwtE5EWKADylUqFu2L79Im7Jd3bqafCM8/AzTfD//4vjBkT7iLeuTPulolIFgVAvsq5CFyh9eoFV10Fq1aFs4DvfCdcOrvnHpWTEOlGFAD5SEIRuK4wYgTMnRvOBIYMgc9/Hk46CR5/PO6WiQgKgPy8/HIY2aIzgI45/fQwRPTuu0NBvdNPh898prljXURioQDIh0YAdV6PHuEMYPXqMOPYwoVhtNCXvgTr18fdOpFEUgDkI4lF4LpKnz7w7W+H0UFXXAH33RdGVn31q7A5e54hEelKCoB8pFJw6KEqg1xIH/pQGCm0Zk24f+CXv4Qjjwydx6+8EnfrRBJBAZCPhobwV2pSi8B1peHD4ec/D5eGpkyBn/4UqqrgX/9V9xCIdDEFQD5SKV3+6WrV1XDHHeHf+pJLwvOaGrjoonCXsYgUnAKgLTt3wksvqQO4WEaPhttvD/MRX3UVPPwwHHssTJwIf/6zykuIFJACoC0qAhePYcNCXaGXXoLvfhf+8hf42Megri4MJ1X5aZFOUwC0RUNA4zVgAFx/fQiCmTPhvfdCCeojjgglJrZsibuFIiVLAdCWhobQ+VtTE3dLkq1PH/iXfwn9AX/8I5x4YigxcfjhIRCeeEKXh0TaSQHQFhWB617M4Oyz4Xe/C+E8dWooPX3qqWHC+ltugW3b4m6lSElQALRFReC6r6OPhltvDaU6Zs6E3r3DZPXDhoU7jJcu1VmBSCvyCgAzG29mKTNrNLPrcqyvMLP7o/VLzawqY930aHnKzM6Nlh1kZsvM7K9mtsLMvluoAyooFYErDYceGi4P1deHx+c/H4rQnXwyfPjDoTNZN5eJHKDNADCznsCtwHlALTDFzGqzNpsKbHX30cAM4KZo31pgMjAGGA/cFr3fLuBMdz8OOB4Yb2YnF+aQCmjzZnjnHZ0BlJITTwzDSF95BX7xixAO11wTKpOedx7Mnh3mdxaRvM4AxgGN7r7W3XcDc4CJWdtMBO6Kns8DzjIzi5bPcfdd7r4OaATGeZCeOPYD0aP7natrBFDp6tsXvvIVePLJ8DlOnw4rV8KFF8Jhh8GXvxzKVGt+AkmwfAJgOLAx4/WmaFnObdx9D7AdGNjavmbW08yeA7YAj7r70ly/3MymmVm9mdU3NTXl0dwCSheBUwCUtqOOgu9/P9xctmhRmK5yzhz4+Mdh5Ei48soQFOovkISJrRPY3fe6+/HACGCcmf1jC9vNdPc6d68bPHhwcRupInDlpUcPOOOMUGbitdfC5aBx48Kloo9+NNQg+vd/h6efVhhIIuQTAJuBkRmvR0TLcm5jZr2ASuCNfPZ1923AYkIfQfeS7gBWEbjyc8ghMHlyGEK6ZUu4u/jYY0MxunHjQkmKa68NZwa6TCRlKp8AeBqoMbNqM+tN6NSdn7XNfODi6PkkYJG7e7R8cjRKqBqoAZaZ2WAz6wdgZgcDZwMNnT+cAtMQ0GT44AfDyKFHHglnBr/+dbjx7+abw5nB8OEwbRr8/vfhTmSRMtFmAETX9C8DFgKrgLnuvsLMbjCzCdFms4CBZtYIXA1cF+27ApgLrAQWAJe6+15gKLDYzJ4nBMyj7v5IYQ+tk1QELpn694cvfhEWLICmJrj33jCF5ezZcP75MGgQfPazYYL7rVvjbq1Ip5iX0LXOuro6r6+vL84ve+45OOGEMJ78s58tzu+U7mvXLli8OFQnnT8/DDPt2TPcgTx+fBhietxxulwo3Y6ZLXf3ulzrdCdwSzQEVDJVVIQv+l/8AjZtCncZX3stvPUWfPOb4Y+F4cPDHcgPPKCzAykJveJuQLelInDSkh49QkfxuHHwgx+Es4GFC+EPfwidynfcEc4OTj45nBmccw6MHRuWiXQjugTUkgsvDCNA1q0rzu+T8rBnTzg7+MMfQj/C8uVheWVluO/grLPgzDOhtlaXi6QoWrsEpABoydixMGRI+B9ZpKNeey3cfLZoETz2WPMfFIcdFoIgHQhVVbE2U8pXawGgS0C57NsX+gBOPz3ulkipGzIkTHY/ZUp4nb4bOR0I990Xlo8aFf57O+208Bg9WmcI0uUUALls3hyGgaoDWAqtujrMYTB1arjbeNWqEASLFoX7EO68M2w3ZEhzGJx2Wqhqqj4EKTAFQC7pEUC6CUy6klnoC6itDfMYuIfBB48/3vyYNy9s27dvuCnttNPC0NO6ulCmRKQTFAC5qAicxMEM/uEfwmPatLBs48bmMFiyBL797bC8R49QuuKkk8Joo5NOCn+w9NDIbsmfAiCXVCr8xTV0aNwtkaQbOTKMSLvwwvD6zTfDKKOnngqPuXPDbGgQRhqNG7d/KAwaFF/bpdtTAOSiInDSXQ0YEO4tOO+88HrfPli9OoRBOhhuvBH27g3rR40Kk+SMHdv8c+DA+Nov3YoCIJeGBvjYx+JuhUjbevQIl36OOQYuuSQse+edcP/BU0/BsmVhmswHHmjep6qqORDSoVDsUuvSLSgAsr3zTrjuquv/UqoOOSQMKc0cxvzmm/DMM+GxfHl4PPhg8/qRI5vD4LjjwqijI47QWXCZUwBkW706/NQIICknAwbAJz4RHmnbtsGzzzYHwjPPhGJ3aR/8YAiCzMexx2r0URlRAGRTEThJin79wgxpZ5zRvGzHDnjhBXj++fD461/hN78Jy9OOPPLAYKiu1n0KJUgBkC1dBG706LhbIlJ8ffvCKaeER5o7bNjQHAjpcHj44eapMw86KJw119bCmDHN9zeMGgW99DXTXemTyZZKhU6ygw+OuyUi3YNZ+H+iqgomTGhevnMnrFgRwmDlyvBYsqS5vAVA797hbDodCOnH6NFhncRKAZCtoUGXf0Ty0acPfOQj4ZFpx47w/1E6FFauhKefDvcspM8YevUKpdaPPhqOOqr5cfTRYUSSOp+LQgGQKT2m+uMfj7slIqWrb9/cwbBzZzjDzgyG1avDXMu7dzdvV1m5fyCkn9fUqAO6wBQAmTZtUhE4ka7Sp0+YOe2EE/ZfvndvmH87lQqBkH48/niYkznT8OHNgTB6dOhjOPLI8LNv3+IdS5lQAGRSETiR4uvZM4wiqq4O025mevddaGwMgZAZEA88EO5tyDR4cAiDdCBkPh86VJeVclAAZNIQUJHu5eCDw70Hxx574Lpt22DtWlizJjzSz5csgdmzwyXdzPcZNerAYDjiiPBI6KUlBUCmhoZw88thh8XdEhFpS79+4c7lsWMPXLd7dxi6mh0Oa9aE+Rd27tx/+4EDwyindCCkn6d/9utXhAMqPgVAJhWBEykPvXuHTuOamgPXuYepOteuDSGxYQOsXx9+rloVpoF9993996ms3D8Qsp8PGlSS3xt5BYCZjQd+AvQEfuXuP8paXwHcDZwIvAFc4O7ro3XTganAXuBr7r7QzEZG2w8BHJjp7j8pyBF1RkPD/ndFikj5MQtn+YcdFibZyeYOr7/eHArpnxs2hCk9Fy/e/85ogIoKGDEiPEaO3P9n+nk3DIk2A8DMegK3AmcDm4CnzWy+u6/M2GwqsNXdR5vZZOAm4AIzqwUmA2OAYcD/mNlRwB7g6+7+jJn1BZab2aNZ71lcb78dRgHp+r9IspmFDuXBgw8cygohILZtaw6FDRvCd8emTaGQ5JIlYVrZv/99//1yhUR2YBQ5JPI5AxgHNLr7WgAzmwNMBDK/rCcC/xE9nwfcYmYWLZ/j7ruAdWbWCIxz9yeBVwDcfYeZrQKGZ71ncaWLwCkARKQ1ZtC/f3gcf3zubfbtgy1bQiCkgyHzZ2shMWxYGO6a/jl8eAiHz32u4IeSTwAMBzZmvN4EnNTSNu6+x8y2AwOj5U9l7Ts8c0czqwJOAJbm+uVmNg2YBnD44Yfn0dwO0hBQESmUHj2aLzPlOouAlkNi8+bwePZZeOSR0GE9dGhsAdBlzOxQ4LfAle7+Vq5t3H0mMBOgrq7Ou6wxKgInIsWUT0i4w/btB97zUCD5BMBmYGTG6xHRslzbbDKzXkAloTO4xX3N7AOEL/973f1B4pZKhRtRDjoo7paIiARmYQhqFw1D7ZHHNk8DNWZWbWa9CZ2687O2mQ9cHD2fBCxyd4+WTzazCjOrBmqAZVH/wCxglbvfXIgD6TQVgRORhGkzANx9D3AZsBBYBcx19xVmdoOZpWvDzgIGRp28VwPXRfuuAOYSOncXAJe6+17gVODzwJlm9lz0+GSBjy1/6SJwuv4vIgmSVx+Au/8e+H3Wsusznr8HfLaFfX8A/CBr2RKg+wyI3bgx3PihMwARSZB8LgGVP40AEpEEUgCAisCJSCIpACB0AFdWwpAhcbdERKRoFACgInAikkgKANAQUBFJJAXAjh3htmt1AItIwigAVARORBJKAaAhoCKSUAqAhoZQlElF4EQkYRQA6SJwFRVxt0REpKgUABoBJCIJlewA2LcPXnxR1/9FJJGSHQAqAiciCZbsAGhoCD91BiAiCZTsAFAROBFJsGQHQLoI3Ic+FHdLRESKLtkBkEqFyz8qAiciCZTsANAQUBFJsOQGwI4d8PLL6gAWkcRKbgCoA1hEEk4BoDMAEUmo5AZAugjckUfG3RIRkVjkFQBmNt7MUmbWaGbX5VhfYWb3R+uXmllVxrrp0fKUmZ2bsfzXZrbFzF4oxIG0WyoFo0apCJyIJFabAWBmPYFbgfOAWmCKmdVmbTYV2Oruo4EZwE3RvrXAZGAMMB64LXo/gDujZfHQCCARSbh8zgDGAY3uvtbddwNzgIlZ20wE7oqezwPOMjOLls9x913uvg5ojN4Pd/8z8GYBjqH90kXgFAAikmD5BMBwYGPG603RspzbuPseYDswMM99W2Vm08ys3szqm5qa2rNry156Cd57Tx3AIpJo3b4T2N1nunudu9cNHjy4MG+aLgKnMwARSbB8AmAzMDLj9YhoWc5tzKwXUAm8kee+xachoCIieQXA00CNmVWbWW9Cp+78rG3mAxdHzycBi9zdo+WTo1FC1UANsKwwTe+Ehgbo1w8KdUYhIlKC2gyA6Jr+ZcBCYBUw191XmNkNZjYh2mwWMNDMGoGrgeuifVcAc4GVwALgUnffC2Bms4EngaPNbJOZTS3sobVCReBERLDwh3ppqKur8/r6+s6/0bBhcM45cOednX8vEZFuzMyWu3tdrnXdvhO44N56C155Rdf/RSTxkhcAKgInIgIkOQB0BiAiCZe8AGhogJ49VQRORBIveQGQSkF1NfTuHXdLRERilcwA0OUfEZGEBcDevbB6tTqARURIWgC89BLs2qUzABERkhYAKgInIvK+ZAWAhoCKiLwvWQHQ0AD9+8OgQXG3REQkdskKABWBExF5X7ICQPMAi4i8LzkBsH07vPqqrv+LiESSEwAqAicish8FgIhIQiUrAFQETkTkfckJgIYGGDVKReBERCLJCQAVgRMR2U8yAmDvXnjxRV3/FxHJkIwA2LBBReBERLIkIwBUBE5E5ADJCAAVgRMROUBeAWBm480sZWaNZnZdjvUVZnZ/tH6pmVVlrJseLU+Z2bn5vmdBNTTAgAEqAicikqHNADCznsCtwHlALTDFzGqzNpsKbHX30cAM4KZo31pgMjAGGA/cZmY983zPwtEIIBGRA+RzBjAOaHT3te6+G5gDTMzaZiJwV/R8HnCWmVm0fI6773L3dUBj9H75vGfhqAiciMgB8gmA4cDGjNebomU5t3H3PcB2YGAr++bzngCY2TQzqzez+qampjyam2XPHjj3XDjzzPbvKyJSxnrF3YC2uPtMYCZAXV2dt/sNevWCu+5qezsRkYTJ5wxgMzAy4/WIaFnObcysF1AJvNHKvvm8p4iIdKF8AuBpoMbMqs2sN6FTd37WNvOBi6Pnk4BF7u7R8snRKKFqoAZYlud7iohIF2rzEpC77zGzy4CFQE/g1+6+wsxuAOrdfT4wC/iNmTUCbxK+0Im2mwusBPYAl7r7XoBc71n4wxMRkZZY+EO9NNTV1Xl9fX3czRARKRlmttzd63KtS8adwCIicgAFgIhIQikAREQSSgEgIpJQJdUJbGZNwIYO7j4IeL2AzSkFOuZk0DGXv84c7xHuPjjXipIKgJ5kABkAAAV4SURBVM4ws/qWesLLlY45GXTM5a+rjleXgEREEkoBICKSUEkKgJlxNyAGOuZk0DGXvy453sT0AYiIyP6SdAYgIiIZFAAiIglV9gFQ1Mnni8jMRprZYjNbaWYrzOyKaPkAM3vUzF6MfvaPlpuZ/TT6d3jezMbGewQdF80r/ayZPRK9rjazpdGx3R+VGCcqQ35/tHypmVXF2e6OMrN+ZjbPzBrMbJWZnVLun7OZXRX9d/2Cmc02s4PK7XM2s1+b2RYzeyFjWbs/VzO7ONr+RTO7ONfvaklZB0DRJ58vrj3A1929FjgZuDQ6tuuAx9y9Bngseg3h36AmekwDfl78JhfMFcCqjNc3ATPcfTSwFZgaLZ8KbI2Wz4i2K0U/ARa4+zHAcYRjL9vP2cyGA18D6tz9Hwkl4ydTfp/zncD4rGXt+lzNbADwHeAkwlzr30mHRl7cvWwfwCnAwozX04Hpcberi471/wFnAylgaLRsKJCKnt8OTMnY/v3tSulBmD3uMeBM4BHACHdI9sr+zAnzTZwSPe8VbWdxH0M7j7cSWJfd7nL+nGmeM3xA9Lk9Apxbjp8zUAW80NHPFZgC3J6xfL/t2nqU9RkA7Zh8vpRFp7wnAEuBIe7+SrTqVWBI9Lxc/i3+C/gGsC96PRDY5u57oteZx/X+MUfrt0fbl5JqoAm4I7rs9SszO4Qy/pzdfTPwY+Al4BXC57ac8v6c09r7uXbq8y73ACh7ZnYo8FvgSnd/K3Odhz8Jymacr5l9Ctji7svjbksR9QLGAj939xOAd2i+LACU5efcH5hICL9hwCEceKmk7BXjcy33ACjryefN7AOEL/973f3BaPFrZjY0Wj8U2BItL4d/i1OBCWa2HphDuAz0E6CfmaWnN808rvePOVpfCbxRzAYXwCZgk7svjV7PIwRCOX/OnwDWuXuTu/8deJDw2Zfz55zW3s+1U593uQdA2U4+b2ZGmIt5lbvfnLFqPpAeCXAxoW8gvfwL0WiCk4HtGaeaJcHdp7v7CHevInyWi9z9/wCLgUnRZtnHnP63mBRtX1J/Kbv7q8BGMzs6WnQWYY7tsv2cCZd+TjazPtF/5+ljLtvPOUN7P9eFwDlm1j86czonWpafuDtBitDJ8klgNbAG+Fbc7Sngcf0T4fTweeC56PFJwrXPx4AXgf8BBkTbG2FE1Brgb4QRFrEfRyeO/+PAI9HzUcAyoBF4AKiIlh8UvW6M1o+Ku90dPNbjgfros34Y6F/unzPwXaABeAH4DVBRbp8zMJvQx/F3wpne1I58rsCXomNvBL7YnjaoFISISEKV+yUgERFpgQJARCShFAAiIgmlABARSSgFgIhIQikARFphZt+KqlI+b2bPmdlJZnalmfWJu20inaVhoCItMLNTgJuBj7v7LjMbBPQGniCMw3491gaKdJLOAERaNhR43d13AURf+JMI9WkWm9liADM7x8yeNLNnzOyBqD4TZrbezP6vmf3NzJaZ2ei4DkQkFwWASMv+CIw0s9VmdpuZfczdfwq8DJzh7mdEZwXfBj7h7mMJd+xenfEe2939WOAWQiVTkW6jV9ubiCSTu79tZicCpwFnAPfbgbPKnUyYbOgvoWwNvYEnM9bPzvg5o2tbLNI+CgCRVrj7XuBPwJ/M7G80F+pKM+BRd5/S0lu08FwkdroEJNICMzvazGoyFh0PbAB2AH2jZU8Bp6av75vZIWZ2VMY+F2T8zDwzEImdzgBEWnYo8DMz60eYg7mRMB/rFGCBmb0c9QNcAsw2s4pov28TKtAC9Dez54Fd0X4i3YaGgYp0kWjiGg0XlW5Ll4BERBJKZwAiIgmlMwARkYRSAIiIJJQCQEQkoRQAIiIJpQAQEUmo/w9HmJlb9SzBEgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KMtlbz8YMd_j",
        "colab_type": "text"
      },
      "source": [
        "## Train"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4DadTw6RMhni",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# optimizer의 learning_rate를 InverseSquareRootSchedule로 지정\n",
        "learning_rate = InverseSquareRootSchedule(config.d_model, warmup_steps=500)\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98, epsilon=1e-9)\n",
        "\n",
        "model.compile(loss=lm_loss, optimizer=optimizer, metrics=[lm_acc])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r1YwBGFwMjQ7",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "913a7bac-1be6-4906-ac63-7f74928d5c18"
      },
      "source": [
        "# save weights callback\n",
        "save_weights = tf.keras.callbacks.ModelCheckpoint(\"songys_chat_transformer.hdf5\", monitor=\"lm_acc\", verbose=1, save_best_only=True, mode=\"max\", save_freq=\"epoch\", save_weights_only=True)\n",
        "# train\n",
        "history = model.fit(train_inputs, train_labels, epochs=50, batch_size=256, callbacks=[save_weights])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/50\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/indexed_slices.py:432: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
            "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "47/47 [==============================] - ETA: 0s - loss: 1.6976 - lm_acc: 0.1412\n",
            "Epoch 00001: lm_acc improved from -inf to 0.14118, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 7s 156ms/step - loss: 1.6976 - lm_acc: 0.1412\n",
            "Epoch 2/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 1.3716 - lm_acc: 0.2881\n",
            "Epoch 00002: lm_acc improved from 0.14118 to 0.28814, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 7s 159ms/step - loss: 1.3716 - lm_acc: 0.2881\n",
            "Epoch 3/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 1.2792 - lm_acc: 0.2944\n",
            "Epoch 00003: lm_acc improved from 0.28814 to 0.29435, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 161ms/step - loss: 1.2792 - lm_acc: 0.2944\n",
            "Epoch 4/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 1.2140 - lm_acc: 0.3075\n",
            "Epoch 00004: lm_acc improved from 0.29435 to 0.30751, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 161ms/step - loss: 1.2140 - lm_acc: 0.3075\n",
            "Epoch 5/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 1.1622 - lm_acc: 0.3223\n",
            "Epoch 00005: lm_acc improved from 0.30751 to 0.32226, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 1.1622 - lm_acc: 0.3223\n",
            "Epoch 6/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 1.0677 - lm_acc: 0.3478\n",
            "Epoch 00006: lm_acc improved from 0.32226 to 0.34781, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 165ms/step - loss: 1.0677 - lm_acc: 0.3478\n",
            "Epoch 7/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.9386 - lm_acc: 0.3891\n",
            "Epoch 00007: lm_acc improved from 0.34781 to 0.38907, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 166ms/step - loss: 0.9386 - lm_acc: 0.3891\n",
            "Epoch 8/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.8041 - lm_acc: 0.4398\n",
            "Epoch 00008: lm_acc improved from 0.38907 to 0.43983, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 167ms/step - loss: 0.8041 - lm_acc: 0.4398\n",
            "Epoch 9/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.6843 - lm_acc: 0.4920\n",
            "Epoch 00009: lm_acc improved from 0.43983 to 0.49197, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 165ms/step - loss: 0.6843 - lm_acc: 0.4920\n",
            "Epoch 10/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.5840 - lm_acc: 0.5401\n",
            "Epoch 00010: lm_acc improved from 0.49197 to 0.54014, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 164ms/step - loss: 0.5840 - lm_acc: 0.5401\n",
            "Epoch 11/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.5099 - lm_acc: 0.5776\n",
            "Epoch 00011: lm_acc improved from 0.54014 to 0.57758, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 0.5099 - lm_acc: 0.5776\n",
            "Epoch 12/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.4356 - lm_acc: 0.6193\n",
            "Epoch 00012: lm_acc improved from 0.57758 to 0.61928, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 162ms/step - loss: 0.4356 - lm_acc: 0.6193\n",
            "Epoch 13/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.3677 - lm_acc: 0.6605\n",
            "Epoch 00013: lm_acc improved from 0.61928 to 0.66055, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 164ms/step - loss: 0.3677 - lm_acc: 0.6605\n",
            "Epoch 14/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.3097 - lm_acc: 0.7013\n",
            "Epoch 00014: lm_acc improved from 0.66055 to 0.70126, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 0.3097 - lm_acc: 0.7013\n",
            "Epoch 15/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.2660 - lm_acc: 0.7321\n",
            "Epoch 00015: lm_acc improved from 0.70126 to 0.73214, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 164ms/step - loss: 0.2660 - lm_acc: 0.7321\n",
            "Epoch 16/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.2328 - lm_acc: 0.7586\n",
            "Epoch 00016: lm_acc improved from 0.73214 to 0.75860, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 164ms/step - loss: 0.2328 - lm_acc: 0.7586\n",
            "Epoch 17/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.2013 - lm_acc: 0.7836\n",
            "Epoch 00017: lm_acc improved from 0.75860 to 0.78361, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 164ms/step - loss: 0.2013 - lm_acc: 0.7836\n",
            "Epoch 18/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.1776 - lm_acc: 0.8046\n",
            "Epoch 00018: lm_acc improved from 0.78361 to 0.80460, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 164ms/step - loss: 0.1776 - lm_acc: 0.8046\n",
            "Epoch 19/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.1582 - lm_acc: 0.8208\n",
            "Epoch 00019: lm_acc improved from 0.80460 to 0.82075, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 164ms/step - loss: 0.1582 - lm_acc: 0.8208\n",
            "Epoch 20/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.1424 - lm_acc: 0.8355\n",
            "Epoch 00020: lm_acc improved from 0.82075 to 0.83546, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 164ms/step - loss: 0.1424 - lm_acc: 0.8355\n",
            "Epoch 21/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.1274 - lm_acc: 0.8513\n",
            "Epoch 00021: lm_acc improved from 0.83546 to 0.85128, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 0.1274 - lm_acc: 0.8513\n",
            "Epoch 22/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.1170 - lm_acc: 0.8618\n",
            "Epoch 00022: lm_acc improved from 0.85128 to 0.86182, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 0.1170 - lm_acc: 0.8618\n",
            "Epoch 23/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.1055 - lm_acc: 0.8728\n",
            "Epoch 00023: lm_acc improved from 0.86182 to 0.87284, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 0.1055 - lm_acc: 0.8728\n",
            "Epoch 24/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0967 - lm_acc: 0.8840\n",
            "Epoch 00024: lm_acc improved from 0.87284 to 0.88402, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 0.0967 - lm_acc: 0.8840\n",
            "Epoch 25/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0899 - lm_acc: 0.8907\n",
            "Epoch 00025: lm_acc improved from 0.88402 to 0.89073, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 0.0899 - lm_acc: 0.8907\n",
            "Epoch 26/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0819 - lm_acc: 0.9002\n",
            "Epoch 00026: lm_acc improved from 0.89073 to 0.90023, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 0.0819 - lm_acc: 0.9002\n",
            "Epoch 27/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0799 - lm_acc: 0.9020\n",
            "Epoch 00027: lm_acc improved from 0.90023 to 0.90204, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 164ms/step - loss: 0.0799 - lm_acc: 0.9020\n",
            "Epoch 28/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0733 - lm_acc: 0.9101\n",
            "Epoch 00028: lm_acc improved from 0.90204 to 0.91010, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 0.0733 - lm_acc: 0.9101\n",
            "Epoch 29/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0685 - lm_acc: 0.9148\n",
            "Epoch 00029: lm_acc improved from 0.91010 to 0.91477, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 0.0685 - lm_acc: 0.9148\n",
            "Epoch 30/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0656 - lm_acc: 0.9180\n",
            "Epoch 00030: lm_acc improved from 0.91477 to 0.91798, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 164ms/step - loss: 0.0656 - lm_acc: 0.9180\n",
            "Epoch 31/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0618 - lm_acc: 0.9237\n",
            "Epoch 00031: lm_acc improved from 0.91798 to 0.92368, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 0.0618 - lm_acc: 0.9237\n",
            "Epoch 32/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0579 - lm_acc: 0.9273\n",
            "Epoch 00032: lm_acc improved from 0.92368 to 0.92729, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 0.0579 - lm_acc: 0.9273\n",
            "Epoch 33/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0569 - lm_acc: 0.9291\n",
            "Epoch 00033: lm_acc improved from 0.92729 to 0.92906, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 0.0569 - lm_acc: 0.9291\n",
            "Epoch 34/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0543 - lm_acc: 0.9316\n",
            "Epoch 00034: lm_acc improved from 0.92906 to 0.93161, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 0.0543 - lm_acc: 0.9316\n",
            "Epoch 35/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0500 - lm_acc: 0.9363\n",
            "Epoch 00035: lm_acc improved from 0.93161 to 0.93634, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 0.0500 - lm_acc: 0.9363\n",
            "Epoch 36/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0484 - lm_acc: 0.9397\n",
            "Epoch 00036: lm_acc improved from 0.93634 to 0.93971, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 0.0484 - lm_acc: 0.9397\n",
            "Epoch 37/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0480 - lm_acc: 0.9387\n",
            "Epoch 00037: lm_acc did not improve from 0.93971\n",
            "47/47 [==============================] - 8s 161ms/step - loss: 0.0480 - lm_acc: 0.9387\n",
            "Epoch 38/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0458 - lm_acc: 0.9426\n",
            "Epoch 00038: lm_acc improved from 0.93971 to 0.94265, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 162ms/step - loss: 0.0458 - lm_acc: 0.9426\n",
            "Epoch 39/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0442 - lm_acc: 0.9448\n",
            "Epoch 00039: lm_acc improved from 0.94265 to 0.94480, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 0.0442 - lm_acc: 0.9448\n",
            "Epoch 40/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0405 - lm_acc: 0.9494\n",
            "Epoch 00040: lm_acc improved from 0.94480 to 0.94943, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 0.0405 - lm_acc: 0.9494\n",
            "Epoch 41/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0395 - lm_acc: 0.9505\n",
            "Epoch 00041: lm_acc improved from 0.94943 to 0.95053, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 164ms/step - loss: 0.0395 - lm_acc: 0.9505\n",
            "Epoch 42/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0393 - lm_acc: 0.9508\n",
            "Epoch 00042: lm_acc improved from 0.95053 to 0.95085, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 0.0393 - lm_acc: 0.9508\n",
            "Epoch 43/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0378 - lm_acc: 0.9518\n",
            "Epoch 00043: lm_acc improved from 0.95085 to 0.95178, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 162ms/step - loss: 0.0378 - lm_acc: 0.9518\n",
            "Epoch 44/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0358 - lm_acc: 0.9538\n",
            "Epoch 00044: lm_acc improved from 0.95178 to 0.95377, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 0.0358 - lm_acc: 0.9538\n",
            "Epoch 45/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0360 - lm_acc: 0.9545\n",
            "Epoch 00045: lm_acc improved from 0.95377 to 0.95451, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 162ms/step - loss: 0.0360 - lm_acc: 0.9545\n",
            "Epoch 46/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0339 - lm_acc: 0.9573\n",
            "Epoch 00046: lm_acc improved from 0.95451 to 0.95732, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 0.0339 - lm_acc: 0.9573\n",
            "Epoch 47/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0333 - lm_acc: 0.9572\n",
            "Epoch 00047: lm_acc did not improve from 0.95732\n",
            "47/47 [==============================] - 8s 161ms/step - loss: 0.0333 - lm_acc: 0.9572\n",
            "Epoch 48/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0314 - lm_acc: 0.9604\n",
            "Epoch 00048: lm_acc improved from 0.95732 to 0.96038, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 0.0314 - lm_acc: 0.9604\n",
            "Epoch 49/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0299 - lm_acc: 0.9623\n",
            "Epoch 00049: lm_acc improved from 0.96038 to 0.96230, saving model to songys_chat_transformer.hdf5\n",
            "47/47 [==============================] - 8s 163ms/step - loss: 0.0299 - lm_acc: 0.9623\n",
            "Epoch 50/50\n",
            "47/47 [==============================] - ETA: 0s - loss: 0.0296 - lm_acc: 0.9620\n",
            "Epoch 00050: lm_acc did not improve from 0.96230\n",
            "47/47 [==============================] - 8s 161ms/step - loss: 0.0296 - lm_acc: 0.9620\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zqJUd1RoMsic",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "outputId": "83658b9c-d2e2-46eb-a3d4-eae4d51704a1"
      },
      "source": [
        "# training result\n",
        "plt.figure(figsize=(12, 4))\n",
        "\n",
        "plt.subplot(1, 2, 1)\n",
        "plt.plot(history.history['loss'], 'b-', label='loss')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend()\n",
        "\n",
        "plt.subplot(1, 2, 2)\n",
        "plt.plot(history.history['lm_acc'], 'g-', label='accuracy')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend()\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAskAAAEKCAYAAADgochqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3hUZfrG8e+TRuiQEHqVJiiCEhXFuipiRVdZERVUymJvay/sz7K2lXVdC7IuKrqCZS2oKIK6iwXUoIBARBBBQk3oLYQkz++PM2hESoBJzmRyf67rXGfmzJmZ+ySTkydv3vc95u6IiIiIiMgvEsIOICIiIiISa1Qki4iIiIhsR0WyiIiIiMh2VCSLiIiIiGxHRbKIiIiIyHZUJIuIiIiIbEdFsohIJWFmI81shZnN3MnjZmaPmdk8M5thZoeUd0YRkVihIllEpPJ4Dui5i8dPAdpGlsHAU+WQSUQkJqlIFhGpJNx9ErBqF7v0AkZ5YApQx8walU86EZHYkrS7HcxsJHA6sMLdD9zB4zcCF5R4vQ5AhruvMrMFwHqgCCh098zShKpXr563bNmyVAcgIhJLpk6dmufuGWHn2EtNgEUl7udEti3d1ZN0zhaRimpX5+zdFskE/557HBi1owfd/WHgYQAzOwO4zt1LtlQc7+55exK4ZcuWZGVl7clTRERigpktDDtDeTCzwQRdMmjevLnO2SJSIe3qnL3b7hal+PdcSecDo0u5r4iIxJbFQLMS95tGtv2Gu49w90x3z8zIqKgN5yIiOxe1PslmVo1gQMh/Smx24AMzmxppdRARkdg1FugXmeWiG7DW3XfZ1UJEJF6VprtFaZ0BfLZdV4uj3H2xmdUHJpjZd5GW6d/Y/l93IiISXWY2GjgOqGdmOcBQIBnA3YcD44BTgXnAJuCScJKKiIQvmkVyH7brauHuiyPrFWb2BnAYsMMi2d1HACMAMjMzPYq5RCREW7duJScnh/z8/LCjRFVqaipNmzYlOTk57Cil5u7n7+ZxB66IxnvF6/e9vFTEz5dIvIlKkWxmtYFjgQtLbKsOJLj7+sjtHsDd0Xg/Eak4cnJyqFmzJi1btsTMwo4TFe7OypUrycnJoVWrVmHHiUnx+H0vL/p8icSG3fZJjvx7bjLQ3sxyzGyAmQ0xsyEldjsb+MDdN5bY1gD41MymA18C77r7+9EMLyKxLz8/n/T09LgqlMyM9PR0tZLuQjx+38uLPl8isWG3Lcm7+/dcZJ/nCKaKK7ltPtB5b4OJSPyIx0IpHo8p2vQ12nv62omEL5p9kkMzYwa89BLcdhvUqhV2GhGJNTVq1GDDhg1hxxARkVIqKi4ib1MeuZtyyd2YS+6mXPI25ZG3KY9iLybREklKSCIxIbK2RAYeMpDqKdWjliEuiuT58+HBB+HccyGzVNf0ExER2XeFhYUkJcXFr1KRMrN+y3p+WvsTi9YtYkvhFhIsATML1hgFRQX8uOZHflj1Az+sDpYFaxZQUFSwR+/T58A+KpK317ZtsP7+exXJIrJz7s5NN93Ee++9h5lxxx13cN5557F06VLOO+881q1bR2FhIU899RRHHnkkAwYMICsrCzPj0ksv5brrrgv7EGQPnHXWWSxatIj8/HyuueYaBg8ezPvvv89tt91GUVER9erV48MPP2TDhg1cddVVP3+vhw4dyjnnnPOr/0C89tprvPPOOzz33HNcfPHFpKam8s0339C9e3f69OnDNddcQ35+PlWrVuXZZ5+lffv2FBUVcfPNN/P++++TkJDAoEGDOOCAA3jsscd48803AZgwYQJPPvkkb7zxRphfKpFS2bR1E7kbc1mxccXPy8rNK9m8dTObCzeTX5j/87Jy80p+WvsTC9csZHX+6lK9fs2UmrROa02n+p04q/1ZNK/dnIzqGWRUy/h5nVY1jaSEJIq8iKLiIgqLCynyYF0ntU5UjzcuiuTWrcEM5s4NO4mIxLLXX3+dadOmMX36dPLy8jj00EM55phjeOmllzj55JO5/fbbKSoqYtOmTUybNo3Fixczc+ZMANasWRNyetlTI0eOJC0tjc2bN3PooYfSq1cvBg0axKRJk2jVqhWrVgXT+t9zzz3Url2bb7/9FoDVq3f/Cz0nJ4fPP/+cxMRE1q1bxyeffEJSUhITJ07ktttu4z//+Q8jRoxgwYIFTJs2jaSkJFatWkXdunW5/PLLyc3NJSMjg2effZZLL720TL8OUnkVezHfr/yeb5Z+w/KNy9lQsIH1W9azoWADG7ZuYGPBRgqLC39VaBYVF1FQVMCmrZvYtHUTG7duDNYFG9lStGWn75WUkERqUurPS+0qtWlRpwVHND2CFrVb0Lx2c5rXbk7V5Kq4O8VejBOskxKSaFG7BfWq1St1f/wkSyIpIYkqVInWl+u371Fmr1yOUlOheXMVySKx7tprYdq06L5mly7w6KOl2/fTTz/l/PPPJzExkQYNGnDsscfy1Vdfceihh3LppZeydetWzjrrLLp06cJ+++3H/PnzueqqqzjttNPo0aNHdINXIte+fy3TlkX3G9+lYRce7bnrb/xjjz32cwvtokWLGDFiBMccc8zP06qlpaUBMHHiRMaMGfPz8+rWrbvb9+/duzeJiYkArF27lv79+zN37lzMjK1bt/78ukOGDPm5O8a297vooot48cUXueSSS5g8eTKjRo3ak0MX+RV3Z0PBBpZvXM7yDcuZv3o+U5dOZerSqUxbNo0NBb8ej1ElsQo1UmpQI6UG1ZKrkZKY8qt+vUkJSVRJqkJa1TSqJVf71ZJWNY361euTUS2D+tXrU796fdKrpVMtuRpJCXFRUv5K3BxR27ZBdwsRkT11zDHHMGnSJN59910uvvhirr/+evr168f06dMZP348w4cP55VXXmHkyJFhR5VS+u9//8vEiROZPHky1apV47jjjqNLly589913pX6Nki1a20/HVr36L/0e77zzTo4//njeeOMNFixYwHHHHbfL173kkks444wzSE1NpXfv3urTLEDQ6ruhYANVk6qSnPjri8i4O8s2LGNW7ixmrpjJrBWzyM7LZsn6JSzfuJxNWzf9av9qydXo0rALF3e+mK6Nu3Jww4NpUacF1ZOr/+a1Zefi5iezbVsYPRrcg64XIhJ7StviW1aOPvponn76afr378+qVauYNGkSDz/8MAsXLqRp06YMGjSILVu28PXXX3PqqaeSkpLCOeecQ/v27bnwwgt3/wayQ7tr8S0La9eupW7dulSrVo3vvvuOKVOmkJ+fz6RJk/jxxx9/7m6RlpbGSSedxBNPPMGjkQ/o6tWrqVu3Lg0aNCA7O5v27dvzxhtvULNmzZ2+V5MmTQB47rnnft5+0kkn8fTTT3P88cf/3N0iLS2Nxo0b07hxY+69914mTpxY5l8LiS3rtqxj+rLpzFk5h7kr5zJ3VbD8sOoHNhduBiAlMYXqydWpnlKd6snVyd2Uy6rNq35+jXrV6tExoyNHNjuSBtUb0LBGQxrUCNbNajWjXXo7EhMSwzrEuBE3RXK7drBmDeTlQUZG2GlEJBadffbZTJ48mc6dO2NmPPTQQzRs2JDnn3+ehx9+mOTkZGrUqMGoUaNYvHgxl1xyCcXFxQDcf//9IaeXPdGzZ0+GDx9Ohw4daN++Pd26dSMjI4MRI0bw+9//nuLiYurXr8+ECRO44447uOKKKzjwwANJTExk6NCh/P73v+eBBx7g9NNPJyMjg8zMzJ1OI3jTTTfRv39/7r33Xk477bSftw8cOJDvv/+egw46iOTkZAYNGsSVV14JwAUXXEBubi4dOnQol6+HlD93Z3X+amatmEXWkiymLp1K1pIsvl/5PY4DQTHcum5r2qS1ocd+PWhUsxH5hflsKAj6C2/rN1w3tS4H1j+QA+ofwIH1D6R+9fohH13lYO4edobfyMzM9KysrD16zrvvwumnw2efwZFHllEwEdlj2dnZcVsI7OjYzGyqu1eqeXZ2dM6O5+97NFx55ZUcfPDBDBgwYKf76GsYm9ZtWcfyDctZuXklKzetJG9THis3ryR3Yy4563PIWZfD4nWLyVmX83PLMECTmk3IbJxJZuNMujbqSoeMDjSr1UwtviHb1Tk7rlqSIeiXrCJZRERiVdeuXalevTqPPPJI2FFkF5auX8o3y77hu7zvmJM3h+9WBuvlG5fvcP/khGSa1GpCk5pNOKTRIZzZ/kya1mpKm7Q2ZDbOpGGNhuV8BLKv4qZIbtkSEhM1w4WIiMS2qVOnhh1BtuPuLFy7kP8t+B+TFk5i0k+TmLdq3s+Pp1dNp3299pzW9jTapbejcc3GpFdLp161eqRXDda1qtTS5cTjTNwUycnJ0KqVimQRERHZsU1bN/1yVbcSV3ebnTubnHU5AKRVTePo5kdzWeZlHNbkMPavtz/1qtULObmEIW6KZAi6XGgaOJHY4+5x18ISi+M5Yk08ft/Liz5f+25DwQamLZvG1CVTf543+Lu87yj24p/3qZtal9ZprTm6+dF0b9adY1seS8eMjiRYQojJJVbEVZHcti3897+aBk4klqSmprJy5UrS09PjpmByd1auXElqamrYUWJWPH7fy4s+X3tvyfolvDrrVV6e9TJTcqb8PItEoxqN6Nq4K+d2OJeOGR1pndaa1nVbU7fq7i8cI5VXXBXJ7drBpk2wZAlEpqwUkZA1bdqUnJwccnNzw44SVampqTRt2jTsGDErXr/v5UWfr9LL25THf2b/hzGzxvC/Bf/Dcbo07MKdx9zJoU0OpWujrjSq2SjsmFIBxVWR3LZtsJ47V0WySKxITk7++TLAUnno+y7RlLMuh1HTRzF31VzyNuX9almTvwaA/evtz9Bjh3Legeexf739Q04s8SBui+TdXBVUREREYlhRcREf/PABw6cO553v38HdaVqrKfWq1aNetXq0qtOK9KrpNKrZiNPansZBDQ5S1x6Jqrgqkps1gypVNHhPRESkolq4ZiEvffsSI74ewYI1C6hfvT43d7+ZQYcMolVd/XdCyk9cFcmJidC6taaBExERqSi2FG7h058+5b157zFu7jiy87IBOL7l8Tx44oOctf9ZpCSmhJxSKqO4KpIhGLw3Z07YKURERGRn3J2PF3zMP778BxN+mMDGrRtJSUzhmBbHMPCQgZzR7gzaprcNO6ZUcnFXJLdtC+PGQVFR0LIsIiIisaHYi3nn+3f4yyd/4YvFX9CgegMuOugiTm17Kse3Op4aKTXCjijys90WyWY2EjgdWOHuB+7g8eOAt4AfI5ted/e7I4/1BP4OJALPuPsDUcq9U23bQkEB/PRTcAU+ERERCVdhcSGvzHqF+z+9n5krZtKqTiuGnzac/l36k5qk+aAlNpWmJfk54HFg1C72+cTdTy+5wcwSgSeAk4Ac4CszG+vus/cya6m0axes585VkSwiIhKm/MJ8npv2HA9//jDzV8/ngIwDePHsFznvwPNISoi7f2ZLnNntJ9TdJ5lZy7147cOAee4+H8DMxgC9gDItkktOA9ejR1m+k4iIiOzI2vy1DM8azt+m/I3lG5dzWJPDeKTHI5zZ/kxd8lkqjGj9GXeEmU0HlgB/cvdZQBNgUYl9coDDo/R+O9WoEVSvrmngREREytuyDcv4+5S/82TWk6zbso4erXtw61G3cmyLYzWHsVQ40SiSvwZauPsGMzsVeBPY4yGpZjYYGAzQvHnzvQ5jFrQmaxo4ERGR8jFzxUyGTR7Gv7/9N1uLttL7gN7c3P1mDml0SNjRRPbaPhfJ7r6uxO1xZvakmdUDFgPNSuzaNLJtZ68zAhgBkJmZ6fuSqW1b+PrrfXkFERER2RV3Z+L8iTwy+RHG/zCeqklVGXjwQK7tdq2mb5O4sM9Fspk1BJa7u5vZYUACsBJYA7Q1s1YExXEfoO++vl9ptGsHr78OW7dCcnJ5vKOIiEjlMSVnCkPeGcL05dNpUL0B9x5/L0Myh5BeLT3saCJRU5op4EYDxwH1zCwHGAokA7j7cOBc4DIzKwQ2A33c3YFCM7sSGE8wBdzISF/lMte2bTBP8o8//jLbhYiIiOyb/MJ8hn48lL9O/itNazXlX2f+iws6XUCVpCphRxOJutLMbnH+bh5/nGCKuB09Ng4Yt3fR9t62wvj771Uki4iIREPWkiz6v9mf2bmzGXjwQB45+RFqVakVdiyRMhOXkxSWnAZORERE9l5BUQH3/O8e7v/0fhrWaMh7F7xHzzY9w44lUubiskhOT4c6dTQNnIiIyL6YkjOFwW8P5tsV39K/c38e7fkodVLrhB1LpFzEZZFsFnSzUEuyiIjInlubv5ZbP7yV4VnDaVKrCWP7jOWM9meEHUukXMVlkQxBl4tJk8JOISIiUnG4O6/Nfo2r37+aFRtXcPXhV3PP8fdQs0rNsKOJlLu4vTZku3awaBFs3hx2EhERkdi3aO0izhh9Bn947Q80rtmYLwZ+waM9H1WBLJVWXLckA/zwAxx4YLhZREREYpW78+KMF7nyvSspKi5iWI9hXHX4VSQlxG2JIFIqcduSvK1I1uA9EZFfmFlPM5tjZvPM7JYdPN7czD42s2/MbIaZnRpGTikfuRtzOffVc+n3Zj8OanAQMy6bwXVHXKcCWYRK0JKswXsiIgEzSwSeAE4CcoCvzGysu88usdsdwCvu/pSZdSSY675luYeVMvf2nLcZ+PZA1uSv4aETH+L6I64nMSEx7FgiMSNui+TataF+fbUki4iUcBgwz93nA5jZGKAXULJIdmDbFSJqA0vKNaGUufzCfK4adxXPfPMMnRt0ZuJFE+nUoFPYsURiTtwWyRAM3ptVLhfCFhGpEJoAi0rczwEO326fPwMfmNlVQHXgxPKJJuVhTf4aeo3pxaSFk7il+y38+bg/65LSIjsRt32SAU47Db74AqZMCTuJiEiFcT7wnLs3BU4FXjCz3/yuMLPBZpZlZlm5ubnlHlL2XM66HI5+9mgmL5rM6HNGc/+J96tAFtmFuC6Sr7wSMjLgjjvCTiIiEhMWA81K3G8a2VbSAOAVAHefDKQC9bZ/IXcf4e6Z7p6ZkZFRRnElWmatmMUR/zqChWsW8v6F79PnwD5hRxKJeXFdJNeoAbfeCh9+CB9/HHYaEZHQfQW0NbNWZpYC9AHGbrfPT8AJAGbWgaBIVlNxBfbJwk846tmjKCou4pNLPuF3rX4XdiSRCiGui2SAIUOgcWO4805wDzuNiEh43L0QuBIYD2QTzGIxy8zuNrMzI7vdAAwys+nAaOBid509K6o3v3uTk144iQbVG/D5gM/p3LBz2JFEKoy4HrgHULVq0N3i8sth/Hjo2TPsRCIi4XH3cQTTupXcdleJ27OB7uWdS6Jv3Nxx9H61N10bdeXdvu+SXi097EgiFUrctyQDDBgALVoExbLaQ0REJN5NWjiJc145h071OzH+wvEqkEX2QqUoklNSYOhQmDoV3nor7DQiIiJlZ+qSqZz+0um0rNOS8ReOp3Zq7bAjiVRIlaJIBrjoomDe5DvvhOLisNOIiIhEX3ZuNj3/3ZO0qmlMuGgCGdU184jI3qo0RXJSEvzf/8HMmfDKK2GnERERia4FaxZw0gsnkWiJTOw3kaa1moYdSaRCqzRFMsAf/gAHHhh0vSgsDDuNiIhIdCzbsIwTR53Ipq2bmHDRBNqktQk7kkiFt9si2cxGmtkKM5u5k8cvMLMZZvatmX1uZp1LPLYgsn2amWVFM/jeSEiAe+6B77+HESPCTiMiIrLv1m9Zz6n/PpWlG5Yy7oJxdGrQKexIInGhNC3JzwG7mjjtR+BYd+8E3ANsX34e7+5d3D1z7yJGV69ecOKJ8Kc/wezZYacRERHZe1uLttL71d7MWD6D13q/Rrem3cKOJBI3dlsku/skYNUuHv/c3VdH7k4huMxpzDKDUaOCq/H16QObN4edSEREZM+5O4PeHsT4H8Yz4owRnNL2lLAjicSVaPdJHgC8V+K+Ax+Y2VQzGxzl99prjRoFhfK33wYtyiIiIhXNXR/fxfPTn+fPx/6ZSw++NOw4InEnakWymR1PUCTfXGLzUe5+CHAKcIWZHbOL5w82sywzy8rNzY1WrJ3q2RNuuAGefBLeeKPM305ERCRqns56mns/uZeBBw/krmPv2v0TRGSPRaVINrODgGeAXu6+ctt2d18cWa8A3gAO29lruPsId89098yMjPKZ1/Evf4HMzOCKfD/9VC5vKSIisk/envM2l4+7nFPbnspTpz+FmYUdSSQu7XORbGbNgdeBi9z9+xLbq5tZzW23gR7ADmfICEtKCoweDVu3wgUXaFo4ERGJbd8u/5Y+/+nDIY0O4eVzXyYpISnsSCJxqzRTwI0GJgPtzSzHzAaY2RAzGxLZ5S4gHXhyu6neGgCfmtl04EvgXXd/vwyOYZ+0aQPDh8OnnwbTw4mIiMSi9VvW0/vV3tSqUou3z3+bGik1wo4kEtd2+yeou5+/m8cHAgN3sH0+0Pm3z4g9F1wAEycGRXKnTnDuuWEnEhER+YW7M/idwcxdNZcP+31IwxoNw44kEvf0f5qIJ56AuXODgrlOnWAuZRERkVgwPGs4Y2aO4b7f3cdxLY8LO45IpVCpLku9K9WqwdtvQ/v2cNZZ8NVXYScSERGBqUumcu34azmlzSncctQtYccRqTRUJJdQty68/z7Urw+nnALffRd2IhERqcxWb15N71d706B6A144+wUSTL+2RcqLftq207gxfPABJCXBSSdpajgREQmHu3PJW5ewaN0iXj73ZdKrpYcdSaRSUZG8A23aBC3K69ZBjx6Qlxd2IhERqWz+/sXfeWvOWzx80sMc0eyIsOOIVDoqkneiS5egj/LChdC3b9hpRESkMlmyfgl3fHQHp7U9jWsOvybsOCKVkorkXTjmmOCqfBMmwP/+F3YaERGpLG798Fa2Fm/lsVMe0xX1REKiInk3hgyBhg1h6NCwk4iISGXw5eIvGTV9FNd3u5796u4XdhyRSktF8m5UrQq33BK0JH/8cdhpREQknrk717x/DQ1rNOS2o28LO45IpaYiuRQGDw5mvRg6FNzDTiMiIvHqpW9fYkrOFO4/4X5qVqkZdhyRSk1FcilUrQq33gqffAIffRR2GhERiUcbCzZy88SbyWycSb/O/cKOI1LpqUgupYEDoUkTtSaLiEjZePCzB1m8fjF/7/l3XTREJAbop7CUUlPhttvgs89g4sSw04iISDxZuGYhD3/+MH079eXIZkeGHUdEUJG8RwYMgGbN1JosIiLRddPEmzCMB054IOwoIhKhInkPVKkStCZPnhxculpERGRfTVo4iVdmvcLN3W+mWe1mYccRkQgVyXvo0kuheXO1JouIyL4r9mKuG38dzWo148buN4YdR0RKUJG8h1JS4Pbb4YsvgivxiYiI7K1R00fx9dKveeDEB6iWXC3sOCJSgorkvXDxxVCvHvzzn2EnERGRimpDwQZu/fBWujXtxvkHnh92HBHZjorkvZCSAn37wtixsHp12GlERKQievDTB1m2YRl/O/lvmFnYcURkOyqS91K/flBQAK+8EnYSERGpaH5a+xN/nfxX+nbqS7em3cKOIyI7oCJ5Lx1yCBxwADz/fNhJRESkorll4i0A3H/C/SEnEZGdKVWRbGYjzWyFmc3cyeNmZo+Z2Twzm2Fmh5R4rL+ZzY0s/aMVPGxmQWvy5Mkwd27YaUREpKKYvGgyo2eO5sYjb6R57eZhxxGRnShtS/JzQM9dPH4K0DayDAaeAjCzNGAocDhwGDDUzOrubdhYc+GFkJAAo0aFnURERCqCbVO+NarRiJu63xR2HBHZhVIVye4+CVi1i116AaM8MAWoY2aNgJOBCe6+yt1XAxPYdbFdoTRuDCeeCC+8AMXFYacREZFYN/rb0Xyx+AvuP+F+aqTUCDuOiOxCtPokNwEWlbifE9m2s+1xo39/WLgQJk0KO4mIiMSyTVs3ccuHt9C1UVcu6nxR2HFEZDdiZuCemQ02sywzy8rNzQ07TqmddRbUrKkuFyJSMZhZTzObExlDcstO9vmDmc02s1lm9lJ5Z4xXwyYPI2ddDn87+W8kWMz8+hWRnYjWT+lioOQF55tGtu1s+2+4+wh3z3T3zIyMjCjFKnvVqkHv3vDqq7BxY9hpRER2zswSgScIxpF0BM43s47b7dMWuBXo7u4HANeWe9A4tHT9Uh749AHO6XAOR7c4Ouw4IlIK0SqSxwL9IrNcdAPWuvtSYDzQw8zqRgbs9Yhsiyv9+sGGDfDmm2EnERHZpcOAee4+390LgDEEY0pKGgQ8ERlHgruvKOeMcenOj++koKiAB098MOwoIlJKpZ0CbjQwGWhvZjlmNsDMhpjZkMgu44D5wDzgn8DlAO6+CrgH+Cqy3B3ZFleOPhpatlSXCxGJeaUZJ9IOaGdmn5nZFDOLm8HWYZm+bDojvxnJ1YdfTeu01mHHEZFSSirNTu6+y4vKu7sDV+zksZHAyD2PVnEkJMBFF8F998HixdAkroYmikglk0QwnedxBF3kJplZJ3dfU3InMxtMMOUnzZtrrt+dcXeu/+B60qqmcccxd4QdR0T2gEYOREm/fsE0cP/+d9hJRER2qjTjRHKAse6+1d1/BL4nKJp/paKOIylv7859l49+/Ig/H/dn6qTWCTuOiOwBFclR0qYNHHlkcJlq97DTiIjs0FdAWzNrZWYpQB+CMSUlvUnQioyZ1SPofjG/PEPGi61FW/nTB3+ifXp7/tj1j2HHEZE9pCI5ivr1g9mzYdq0sJOIiPyWuxcCVxIMoM4GXnH3WWZ2t5mdGdltPLDSzGYDHwM3uvvKcBJXbE9PfZo5K+fw1x5/JTkxOew4IrKHVCRHUe/ekJysLhciErvcfZy7t3P31u5+X2TbXe4+NnLb3f16d+/o7p3cfUy4iSum1ZtX8+f//pkTWp3AaW1PCzuOiOwFFclRlJYGp5wCo0dDUVHYaUREJCz3TLqHVZtX8UiPRzCzsOOIyF5QkRxlffvCkiW6TLWISGX1zdJveOyLxxjcdTCdG3YOO46I7CUVyVF2xhlQo4a6XIiIVEZFxUUMfmcw9arV4/4T7g87jojsAxXJUVatGvz+9/Daa5CfH3YaEcQbRZ4AAB/OSURBVBEpT0989QRZS7J4tOej1K1aN+w4IrIPVCSXgb59Ye1aeO+9sJOIiEh5yVmXw+0f3U7PNj0574Dzwo4jIvtIRXIZOOEEqF9fXS5ERCqTq9+7mqLiIp489UkN1hOJAyqSy0BSEvTpA++8E7Qoi4hIfHvru7d447s3GHrsUFrVbRV2HBGJAhXJZaRvX9iyBV5/PewkIiJSltZvWc+V711Jp/qduP6I68OOIyJRoiK5jBx2GLRurS4XIiLx7s6P72TxusWMOGOErqwnEkdUJJcRs6A1+aOPgnmTRUQk/kxfNp1/fPkPLsu8jG5Nu4UdR0SiSEVyGbrgAnCHl18OO4mIiJSFuyfdTc2Umtx3wn1hRxGRKFORXIbat4euXdXlQkQkHs1cMZPXs1/n6sOvpk5qnbDjiEiUqUguY337wtSpMGdO2ElERCSa/vLJX6iRUoNrDr8m7CgiUgZUJJexPn2C/slqTRYRiR/fr/yel2e9zOWZl5NeLT3sOCJSBlQkl7HGjeHEE+H556GoKOw0IiISDX/55C9USazCDUfeEHYUESkjKpLLwaBB8NNP8MEHYScREZF99ePqH3lxxov8sesfqV+9fthxRKSMqEguB716QUYGjBgRdhIREdlXD3z6AIkJifzpyD+FHUVEylCpimQz62lmc8xsnpndsoPH/2Zm0yLL92a2psRjRSUeGxvN8BVFSgpccgm8/bbmTBYRqcgWrV3Es9OeZcDBA2hSq0nYcUSkDO22SDazROAJ4BSgI3C+mXUsuY+7X+fuXdy9C/APoOTFmDdve8zdz4xi9gpl4MCgT/Kzz4adRERE9tbDnz+M49zc/eawo4hIGStNS/JhwDx3n+/uBcAYoNcu9j8fGB2NcPGkbVv43e/gmWeguDjsNCIisqeWbVjGP7/+J/0O6keLOi3CjiMiZaw0RXITYFGJ+zmRbb9hZi2AVsBHJTanmlmWmU0xs7P2OmkcGDQIFiyAiRPDTiIiInvqr5//lYKiAm49+tawo4hIOYj2wL0+wGvuXnKysxbungn0BR41s9Y7eqKZDY4U01m5ublRjhUbzj4b0tM1gE9EpKJZvXk1w7OGc/6B59MmrU3YcUSkHJSmSF4MNCtxv2lk2470YbuuFu6+OLKeD/wXOHhHT3T3Ee6e6e6ZGRkZpYhV8VSpAhdfDG+9BcuWhZ1GRERK6+mpT7Nx60ZuPPLGsKOISDkpTZH8FdDWzFqZWQpBIfybWSrMbH+gLjC5xLa6ZlYlcrse0B2YHY3gFdWgQVBYCM89F3YSEREpjYKiAv7x5T84ab+T6Nywc9hxRKSc7LZIdvdC4EpgPJANvOLus8zsbjMrOVtFH2CMu3uJbR2ALDObDnwMPODulbpIbt8ejj0W/vlPDeATEakIxswcw5L1S7jhCF1dT6QySSrNTu4+Dhi33ba7trv/5x0873Og0z7ki0uDB8MFF8BHHwWXrBYRkdjk7vz1879yYP0D6dG6R9hxRKQc6Yp7Ifj97yEtTQP4RERi3cT5E/l2xbdc3+16zCzsOCJSjlQkhyA1Ffr3hzfegOXLw04jIiI788jkR2hYoyF9O/UNO4qIlDMVySEZPDgYwPfUU2EnERGRHZm5YibjfxjPVYddRZWkKmHHEZFypiI5JPvvD2eeCY89Bhs2hJ1GRES2N2zyMKolV2NI5pCwo4hICFQkh+jWW2H1avVNFhGJNUvXL+XFGS9ySZdLSKuaFnYcEQmBiuQQdesGxx8PjzwCW7aEnUZERLZ5/MvHKSwu5Npu14YdRURCoiI5ZLfdBkuWwKhRYScRERGAjQUbeSrrKc7a/yxdglqkElORHLITToDMTHjwwWAgn4iIhOu5ac+xOn81fzryT2FHEZEQqUgOmVnQmvzDD/Daa2GnERGp3AqLCxk2ZRjdmnbjyGZHhh1HREKkIjkG9OoFHTrA/ffDry7qLSIi5eo/s//D/NXzubn7zWFHEZGQqUiOAQkJcMstMGMGjBu3+/1FRCT63J2HPn+IduntOLP9mWHHEZGQqUiOEeefDy1awF/+otZkEZEwfPTjR3y99GtuPPJGEky/HkUqO50FYkRyMtx4I3z+OXzySdhpREQqnwc/e5CGNRpy0UEXhR1FRGKAiuQYcumlUL8+3Hdf2ElERCqXb5Z+w4T5E7j28Gt1CWoRAVQkx5SqVYPW5A8+gLFjw04jIvHIzHqa2Rwzm2dmt+xiv3PMzM0sszzzheXhzx+mZkpN/pj5x7CjiEiMUJEcY66+Gjp1gssug7Vrw04jIvHEzBKBJ4BTgI7A+WbWcQf71QSuAb4o34Th+HH1j7w862X+2PWP1EmtE3YcEYkRKpJjTEoKPPMMLFsWzHghIhJFhwHz3H2+uxcAY4BeO9jvHuBBIL88w4Vl2ORhJFqiLkEtIr+iIjkGHXYYXHMNDB8OkyaFnUZE4kgTYFGJ+zmRbT8zs0OAZu7+bnkGC0vepjz+9c2/uPCgC2lSq8nunyAilYaK5Bh1zz3QqhUMGgT5laItR0TCZmYJwDDghlLsO9jMsswsKzc3t+zDlZHHv3yczYWbdQlqEfkNFckxqnp1ePpp+P77oGAWEYmCxUCzEvebRrZtUxM4EPivmS0AugFjdzR4z91HuHumu2dmZGSUYeSys7FgI49/+ThntDuDjhm/6ZotIpVcqYrk3Y2GNrOLzSzXzKZFloElHutvZnMjS/9oho93J50E/fvDQw/B9OlhpxGROPAV0NbMWplZCtAH+HkuHXdf6+713L2lu7cEpgBnuntWOHHL1rDJw1i5eSW3HKUBICLyW7stkks7Ghp42d27RJZnIs9NA4YChxMMGBlqZnWjlr4SGDYM0tJg4EAoLAw7jYhUZO5eCFwJjAeygVfcfZaZ3W1mleo6zEvWL+GBzx7gnA7ncGSzI8OOIyIxqDQtyaUdDb0jJwMT3H2Vu68GJgA99y5q5ZSWBv/4B2Rlwd//HnYaEano3H2cu7dz99bufl9k213u/pvZ2d39uHhtRb7tw9soLC7koZMeCjuKiMSo0hTJux0NHXGOmc0ws9fMbFuft9I+V3ahd2844wy4806YPz/sNCIiFdvUJVN5fvrzXHv4texXd7+w44hIjIrWwL23gZbufhBBa/Hze/oC8TJSuiyYwZNPQlIS/PGP4B52IhGRisnduXb8tdSvXp/bj7k97DgiEsNKUyTvbjQ07r7S3bdE7j4DdC3tc0u8RoUfKV2WmjaFBx+EiRPh+T3+E0RERABem/0an/70Kfccfw+1qtQKO46IxLDSFMm7HA0NYGaNStw9k2BACASDQ3qYWd3IgL0ekW2yF/74R+jeHa6/HpYvDzuNiEjFkl+Yz00Tb+KgBgcx4OABYccRkRi32yK5lKOhrzazWWY2HbgauDjy3FUElzf9KrLcHdkmeyEhIbhk9caNwRX5RESk9B6d8igL1ixgWI9hJCYkhh1HRGKceQx2cM3MzPSsrLgcUB0V994bDOIbOzYY0CciscPMprr7by6+Ec8qwjl72YZltPtHO45vdTxv9Xkr7DgiEiN2dc7WFfcqoJtuggMPhMsvh3Xrwk4jIhL7bvvwNjYXbubhkx4OO4qIVBAqkiuglJSg28XixXDrrWGnERGJbcMmD+PZac9ywxE30C69XdhxRKSCUJFcQR1+OFx9dTA13KRJYacREYlNo6aP4oYPbuDcjudy3+/uCzuOiFQgKpIrsHvvhTZtoF8/WLs27DQiIrHlne/f4dK3LuWEVifw4tkvarCeiOwRFckVWI0a8MILkJMDV10VdhoRkdjx6U+f0vvV3hzc6GDeOO8NqiRVCTuSiFQwKpIruG7d4I47gmL5lVfCTiMiEr4Zy2dw+kun06J2C8b1HUfNKjXDjiQiFZCK5Dhw++1BH+UhQ4LBfCIildWcvDmc/OLJ1KxSkw8u+oCM6rqCq4jsHRXJcSA5OWhJ3rIFLr4YiovDTiQiUr4Kigr4yyd/ocvTXdhatJUPLvyA5rWbhx1LRCowFclxom1bePRRmDgRHnss7DQiIuXn80Wfc8jTh3D7R7dzervTmXHZDDpkdAg7lohUcCqS48jAgXDmmXDLLTBzZthpRETK1pr8NVz2zmV0H9mddVvW8fb5b/Nq71dpXLNx2NFEJA6oSI4jZvDPf0Lt2tC3L2zcGHYiEZHoKyou4tlvnqXDEx0Y8fUIrut2HbOvmM3p7U4PO5qIxBEVyXGmfv2gf/KsWdC/v/oni0h8+XD+h3Qd0ZVLx15Ki9ot+HLglww7eRg1UmqEHU1E4oyK5DjUowc8/DD85z/BBUdERCq67/K+44zRZ3DiCyeyJn8NY84Zw+QBk+nauGvY0UQkTiWFHUDKxnXXwYwZMHQoHHAAnHNO2IlERPbMsg3L+HD+h7w37z3GzBxDteRqPHDCA1zT7RpSk1LDjicicU5Fcpwyg+HDYc6c4LLVbdpA585hpxIR2bnC4kLen/c+E+dP5MMfP2TmimAEclrVNIZkDuGuY++ifvX6IacUkcpCRXIcS02F11+HQw8NZr346qugz7KISCx64ssnuHb8taQmpXJ086O56KCLOHG/E+nSsAsJpt6BIlK+VCTHuUaN4M034eij4dxzg3mUU1LCTiUi8lvfLPuGhjUa8uM1P6o7hYiETn+aVwKZmfDss/DJJ3DhhVBQEHYiEZHfys7LpmNGRxXIIhITVCRXEn36wCOPwKuvwumnw4YNYScSEfmFu5Odm02HerpSnojEBhXJlcj118PIkfDRR3DCCZCXF3YiEZHAkvVLWF+wnv3r7R92FBERQEVypXPJJcFgvhkzgn7KP/0UdiIRkWAeZEAtySISM0pVJJtZTzObY2bzzOyWHTx+vZnNNrMZZvahmbUo8ViRmU2LLGOjGV72zplnwvjxsGQJdO8O2dlhJxKRyi47LzgRdchQkSwisWG3RbKZJQJPAKcAHYHzzazjdrt9A2S6+0HAa8BDJR7b7O5dIsuZUcot++iYY2DSJNi6FY46Ct59N+xEIlKZZedmU6tKLRrVaBR2FBERoHQtyYcB89x9vrsXAGOAXiV3cPeP3X1T5O4UoGl0Y0pZ6NwZPvsMmjQJBvMNGaIBfSISjuy8YNCemYUdRUQEKF2R3ARYVOJ+TmTbzgwA3itxP9XMssxsipmdtRcZpQy1bh1cZOTGG2HECDj4YJgyJexUIlLZZOdlq6uFiMSUqA7cM7MLgUzg4RKbW7h7JtAXeNTMWu/kuYMjxXRWbm5uNGPJblSpAg89BB9/HMyh3L073HVX0BVDRKSsrclfw7INyzRoT0RiSmmK5MVAsxL3m0a2/YqZnQjcDpzp7lu2bXf3xZH1fOC/wME7ehN3H+Hume6emZGRUeoDkOg59thg1osLL4R77oEjjoBvvw07lYjEu20zW2j6NxGJJaUpkr8C2ppZKzNLAfoAv5qlwswOBp4mKJBXlNhe18yqRG7XA7oDs6MVXqKvdm14/nl47bVgeriuXYOCWa3KIlJWsnMjM1uoJVlEYshui2R3LwSuBMYD2cAr7j7LzO42s22zVTwM1ABe3W6qtw5AlplNBz4GHnB3FckVwDnnwKxZwfquu+Dww2H69LBTiUg8ys7LJiUxhVZ1W4UdRUTkZ0ml2cndxwHjttt2V4nbJ+7keZ8DnfYloIQnIwNGj4Y//AEuuwwyM+G224KlSpWw04lIvMjOy6ZdejuSEkr1K0lEpFzoinuyW2efHbQq9+kDd98N7drB8OGwZcvunysisjvZudnqaiEiMUdFspRKejq88AJ88AE0bhy0LLdtC089pWJZRPZefmE+P675UUWyiMQcFcmyR046CT7/PLisdbNmcPnl0KYNPP64LkQiIntu7sq5FHux5kgWkZijIln2mBn06AGffgoTJkDLlnDVVUEL8xVXwMyZYScUkZ0xs55mNsfM5pnZLTt4/Hozm21mM8zsQzNrUZZ5svOCmS00/ZuIxBoVybLXzODEE2HSpKB1+ayz4F//gk6d4Kij4N//VlcMkVhiZonAE8ApQEfgfDPruN1u3wCZ7n4Q8BrwUFlmys7NxjDap7cvy7cREdljKpJln5kFFx4ZNQoWL4a//hWWLw8uStKgAfTvD++8o4JZJAYcBsxz9/nuXgCMAXqV3MHdP3b3TZG7UwguIFVmsvOyaVmnJVWTq5bl24iI7DEVyRJV6elwww0wZ07QFePss2HsWDjjjKBg7tcP3n4b8vPDTipSKTUBFpW4nxPZtjMDgPfKMlB2Xrb6I4tITFKRLGUiISHoivHss0Gr8rhx8PvfBy3KZ54ZzMH8hz/ASy/BmjVhpxWR7ZnZhUAmwcWidvT4YDPLMrOs3NzcvXqPouIi5uTN0cwWIhKTVCRLmUtJgVNOgZEjYdkyeP99uOAC+OSTYF2/Ppx8cjBDxrRpUFgYdmKRuLUYaFbiftPItl8xsxOB24Ez3X2HHaXcfYS7Z7p7ZkZGxl6FWbh2IVuKtqhIFpGYpMsbSblKSQkK4pNPhiefhC++gDfeCJarrgr2qVYNDj0UunUL+jofcURQSIvIPvsKaGtmrQiK4z5A35I7mNnBwNNAT3dfUZZhsnODmS3U3UJEYpGKZAlNQsIvRfCDD8KPPwZF85QpwTJsGGzdGuzbvj0cfXSwHHMMtGgRDBgUkdJz90IzuxIYDyQCI919lpndDWS5+1iC7hU1gFct+CH7yd3PLIs8mv5NRGKZimSJCWaw337Bcv75wbb8fPj6a/jss2Cauddeg2eeCR5r2hQyM6FzZ+jSJVhUOIvsnruPA8Ztt+2uErdPLK8s2bnZ1K9en7SqaeX1liIipaYiWWJWaioceWSw3HgjFBcHFyr55JPgQibffANvvQXuwf61awdzNLdtG1wFcNu6TRuoWTPcYxGR38rOy1Z/ZBGJWSqSpcJISICDDgqWK64Itm3cGBTO06bB9OnB7fffh6VLf/3cxo2hY0c44IBflg4doE4dtT6LhMHdyc7Lps8BfcKOIiKyQyqSpUKrXh0OPzxYStqwAX74AebNg7lz4bvvYNYs+Oc/YdOmX/YzCwYKVq/+y7pu3aDbR+vWQSv0tnVamgpqkWhZsXEFa/LXaNCeiMQsFckSl2rUCPord+786+3FxbBgQVAwz5kD69YFrdGbNgXrjRshLw8++ii4gmBJ1aoFLdIll0aNguK5du2gVXrbum7dYEnQJIsiO7Rt0J66W4hIrFKRLJVKQsIvAwTPOGPX+27eHMy4MW9esOTkwJIlwfLVV8F68+Zdv1d6OtSrFywZGb9e6tcP1unpUKUKJCf/eqlaNWjZVuu1xCNN/yYisU5FsshOVK0a9GPu2HHHj7sHLdFr1sDatcF62+1Vq4IW6bw8yM0N1t99Fww6XLkyaNEujYSEYNBhrVpBK3WtWkHB3aBBsNSvH6wzMoKW7tTUIPe2dXJy8BqJicF625KUpOJbwpWdl02NlBo0qbmrq2KLiIRHRbLIXjILCtfatffseUVFsHp1UDzn5gZFc0FBsGzd+suyeXNQhG9b1q4NlgULgvmkc3NLX2xvLykp6JJSo0ZQhNeoEbRalyywt61r1fp1i3h6erBUrRpcHCY5OVinpKj4ltLLzstm/3r7Y/rAiEiMUpEsUs4SE38pODvsw3+ai4qCAnv58qBgzs8Pls2bf7ldUBAU0iWXoqKgD/aGDb8s69cH6+1fZ1uhvreXCjcLluTk3xbfqalBYV5y0GT16kHXE/dgKS7+ZV2lSvD4tuJ+2/5JScHXdNs6MTEo2KtV+2XZ9h5Vq6qQjxXZudn8rtXvwo4hIrJTpSqSzawn8HeCKzQ94+4PbPd4FWAU0BVYCZzn7gsij90KDACKgKvdfXzU0otUYomJQXeLsr5kt3tQRG/rPpKXFxTn+flBi/f2reDb5q3eVui6B9tLFvDbCvBNm4JW9ZycXwZPbtkSFLHbuoZsK7QLCoIcRUX7djwJCUGBvm2pUuXX77PtdsuW8N57+/zlkx1Yv2U9i9cv1qA9EYlpuy2SzSwReAI4CcgBvjKzse4+u8RuA4DV7t7GzPoADwLnmVlHoA9wANAYmGhm7dx9H3/NiUh5MQu6XNSqFQx4DJN7UCxv3Bi0fG/cGLRyFxUFy7bbBQW/FOHbiu9Nm37b0r5tKdlqve12w4bhHms821CwgT4H9uGIZkeEHUVEZKdK05J8GDDP3ecDmNkYoBdQskjuBfw5cvs14HELOpr1Asa4+xbgRzObF3m9ydGJLyKViVnQ8lulSjD1nlRMjWo2YvQ5o8OOISKyS6WZxbUJsKjE/ZzIth3u4+6FwFogvZTPFRERERGJKTFzqQMzG2xmWWaWlZubG3YcEREREanESlMkLwaalbjfNLJth/uYWRJQm2AAX2meC4C7j3D3THfPzMjIKF16EREREZEyUJoi+SugrZm1MrMUgoF4Y7fbZyzQP3L7XOAjd/fI9j5mVsXMWgFtgS+jE11EREREpGzsduCeuxea2ZXAeIIp4Ea6+ywzuxvIcvexwL+AFyID81YRFNJE9nuFYJBfIXCFZrYQERERkVhXqnmS3X0cMG67bXeVuJ0P9N7Jc+8D7tuHjCIiIiIi5SpmBu6JiIiIiMQKFckiIiIiItsx33YN2RhiZrnAwj18Wj0grwzixJJ4P8Z4Pz6I/2OM9+OD3R9jC3evVFP07OU5G+L/8xLvxwfxf4zxfnwQ/8e41+fsmCyS94aZZbl7Ztg5ylK8H2O8Hx/E/zHG+/FB5TjG8hLvX8t4Pz6I/2OM9+OD+D/GfTk+dbcQEREREdmOimQRERERke3EU5E8IuwA5SDejzHejw/i/xjj/figchxjeYn3r2W8Hx/E/zHG+/FB/B/jXh9f3PRJFhERERGJlnhqSRYRERERiYq4KJLNrKeZzTGzeWZ2S9h5osHMRprZCjObWWJbmplNMLO5kXXdMDPuCzNrZmYfm9lsM5tlZtdEtsfFMZpZqpl9aWbTI8f3f5Htrczsi8hn9WUzSwk7674ws0Qz+8bM3oncj7fjW2Bm35rZNDPLimyLi89omHTOrnji/ZwNOm/Hw/FF+5xd4YtkM0sEngBOAToC55tZx3BTRcVzQM/ttt0CfOjubYEPI/crqkLgBnfvCHQDroh83+LlGLcAv3P3zkAXoKeZdQMeBP7m7m2A1cCAEDNGwzVAdon78XZ8AMe7e5cSUwjFy2c0FDpnV1jxfs4Gnbfj5fiids6u8EUycBgwz93nu3sBMAboFXKmfebuk4BV223uBTwfuf08cFa5hooid1/q7l9Hbq8n+IFtQpwcowc2RO4mRxYHfge8FtleYY8PwMyaAqcBz0TuG3F0fLsQF5/REOmcXQHF+zkbdN6O7FKhj28n9vozGg9FchNgUYn7OZFt8aiBuy+N3F4GNAgzTLSYWUvgYOAL4ugYI//SmgasACYAPwBr3L0wsktF/6w+CtwEFEfupxNfxwfBL8gPzGyqmQ2ObIubz2hIdM6u4OL1nA06b1Pxjy+q5+ykaKeT8uHubmYVfmoSM6sB/Ae41t3XBX/UBir6Mbp7EdDFzOoAbwD7hxwpaszsdGCFu081s+PCzlOGjnL3xWZWH5hgZt+VfLCif0al/MTLZyWez9mg83YciOo5Ox5akhcDzUrcbxrZFo+Wm1kjgMh6Rch59omZJROcbP/t7q9HNsfVMQK4+xrgY+AIoI6ZbfvjtCJ/VrsDZ5rZAoJ/l/8O+Dvxc3wAuPviyHoFwS/Mw4jDz2g50zm7gqos52zQeTucePsu2ufseCiSvwLaRkZnpgB9gLEhZyorY4H+kdv9gbdCzLJPIv2g/gVku/uwEg/FxTGaWUakJQIzqwqcRNCH72Pg3MhuFfb43P1Wd2/q7i0JfuY+cvcLiJPjAzCz6mZWc9ttoAcwkzj5jIZI5+wKKN7P2aDzdmS3Cnt8ZXHOjouLiZjZqQT9bBKBke5+X8iR9pmZjQaOA+oBy4GhwJvAK0BzYCHwB3fffqBIhWBmRwGfAN/yS9+o2wj6uFX4YzSzgwgGCCQS/DH6irvfbWb7EfwFnwZ8A1zo7lvCS7rvIv+2+5O7nx5Pxxc5ljcid5OAl9z9PjNLJw4+o2HSObviifdzNui8TQU/vrI4Z8dFkSwiIiIiEk3x0N1CRERERCSqVCSLiIiIiGxHRbKIiIiIyHZUJIuIiIiIbEdFsoiIiIjIdlQkS4VlZkVmNq3EcksUX7ulmc2M1uuJiFR2OmdLRaPLUktFttndu4QdQkRESkXnbKlQ1JIsccfMFpjZQ2b2rZl9aWZtIttbmtlHZjbDzD40s+aR7Q3M7A0zmx5Zjoy8VKKZ/dPMZpnZB5ErMImISBTpnC2xSkWyVGRVt/vX3XklHlvr7p2Axwmu7AXwD+B5dz8I+DfwWGT7Y8D/3L0zcAgwK7K9LfCEux8ArAHOKePjERGJZzpnS4WiK+5JhWVmG9y9xg62LwB+5/7/7d0xSl1REAbgf5BXpApimkCKNNmBe1GxklQWIZW4gazCxm0IYhWIrbgA2yfoDkQmhVe4XDCQ4ul7z+9r7pxTndsMc4fhnr6pqlmS2+7eqqr7JJ+7+2HYn3f3p6q6S/JlfA1nVX1Nct7d34b1cZJZd/9a/JsBrB85m1Wjk8y66hfi/zG+u/4xZvgBFkXOZukokllXO6Pn5RD/SbI7xPtJfg/xRZLDJKmqjar6+FqHBCCJnM0S8pXFKvtQVVej9Vl3P/9SaLOqrvPUWdgb9n4kOa2qoyR3SQ6G/Z9JTqrqe566D4dJ5gs/PcD7ImezUswks3aG+bbt7r5/67MA8G9yNsvKuAUAAEzoJAMAwIROMgAATCiSAQBgQpEMAAATimQAAJhQJAMAwIQiGQAAJv4Cj6qbG035XkgAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 864x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dtqNHlkRdjz5",
        "colab_type": "text"
      },
      "source": [
        "## Inference"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0wV723x3dsOy",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 312
        },
        "outputId": "a290660f-d083-463a-c693-81797c601a2e"
      },
      "source": [
        "model = build_model(config)\n",
        "tf.keras.utils.plot_model(model, 'model.png', show_shapes=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuMAAAEnCAYAAAAD7E2bAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nOzde1hU1f4/8Pcol2GQq3ETJbkogmJ5DUj0GFleQryL5fFHdTqoncNYdDIxCy1U0mNoardDdjRTQXtEU6yjRmolWkoQBQlJqaSgyH2UAdbvDx/m6wgMAwxsBt6v5+GP1qxZ67P3dj592OxZSyaEECAiIiIioo6W1EPqCIiIiIiIuisW40REREREEmExTkREREQkERbjREREREQSMbm34bvvvsOGDRukiIWIiAAkJSW1y7jM70RE0mosvze4M37p0iXs3bu3QwIiMmanT5/G6dOnpQ7DqFy+fJn5RYf2Pj/M70T6YX5vOeZ33XSdnwZ3xuu1150Zoq5i9uzZAPhZaYnExETMnTuX56wJ9eenvfH8E+nG/N5yzO+66crvfGaciIiIiEgiLMaJiIiIiCTCYpyIiIiISCIsxomIiIiIJMJinIiIiIhIIizGiSR2+PBh2NjY4ODBg1KH0iktXLgQMplM8zN//vwGfY4ePYply5ahrq4O06dPh5ubG+RyOVxdXREaGoqMjIwWzdncOAcOHEBcXBxqa2u13rd//36tWO+7777WHzgRGT3md92Y3+9gMU4kMSGE1CF0evb29khJSUFOTg4SEhK0Xnv99dexadMmREdHo66uDidPnsSnn36K4uJinDp1CiqVCmPHjkVBQYHe8zU3ztSpUyGXyxEcHIySkhLN+0JDQ3H58mWcOHECkydPNtjxE5FxYn5vHvM7i3EiyU2ZMgWlpaUICQmROhSoVCoEBgZKHUYDFhYWmDhxIgYOHAhzc3NN+9q1a7F7924kJibCysoKABAQEIAxY8ZAoVDA3d0dsbGxKC0txccff9yiOZsbR6lU4oEHHsDkyZNRU1MDAJDJZHB1dUVQUBAGDBhgkGMnIuPF/N485ncW40R0l4SEBBQWFkodhl5yc3OxYsUKrFy5EnK5HABgYmLS4M/BHh4eAIC8vDy9x9Z3nJiYGKSnpyM+Pr5Vx0BE1FGY39GicToyv7MYJ5LQqVOn4ObmBplMhs2bNwMAtm7dCktLSygUCiQnJ2PSpEmwtrZG3759sWvXLs17N23aBLlcDkdHRyxcuBAuLi6Qy+UIDAxEWlqapl9kZCTMzMzg7OysaXv++edhaWkJmUyG69evAwCWLFmCqKgo5OXlQSaTwcvLCwBw5MgRWFtbIzY2tiNOid42bdoEIQSmTp2qs59KpQIAWFtbt2m+xsaxs7PDuHHjEB8fzz9HE5EW5vfW6275ncU4kYTGjBmDb7/9Vqtt8eLFeOGFF6BSqWBlZYU9e/YgLy8PHh4eeO6556BWqwHcScLh4eGoqqqCUqlEfn4+zp07h5qaGkyYMAGXLl0CcCepzZkzR2uOLVu2YOXKlVpt8fHxCAkJgaenJ4QQyM3NBQDNl1jq6ura5Ry01qFDh+Dt7Q2FQqGz35kzZwDcOddt0dQ4w4YNw5UrV/Djjz+2aXwi6lqY31uvu+V3FuNEnVhgYCCsra3h4OCAsLAwVFZW4o8//tDqY2JiAh8fH5ibm8PX1xdbt25FeXk5tm3bZpAYpkyZgrKyMqxYscIg4xlCZWUlLl68CE9Pzyb7XLt2Dbt374ZSqURAQECzd1haO079s4OZmZmtGp+Iuifm98Z1x/xu0q6jE5HBmJmZAYDmzklTRo4cCYVCgezs7I4ISxKFhYUQQui8axIQEIDKykrMmTMHb775JkxNTVs1V3Pj1Mdw7dq1Vo1PRMT8/n+6Y35nMU7UBZmbm6OoqEjqMNrNrVu3AEDrm/f3cnR0REJCAgYPHtymuZobx8LCQismIqL2xPze9fI7H1Mh6mLUajVKSkrQt29fqUNpN/UJ8t5NGe7m4OAAW1vbNs/V3DjV1dVaMRERtRfm9zu6Wn7nnXGiLiY1NRVCCPj7+2vaTExMmv3zpzFxdHSETCZDaWlpk30MteNdc+PUx+Dk5GSQ+YiImsL8fkdXy++8M05k5Orq6nDz5k3U1NQgIyMDS5YsgZubG8LDwzV9vLy8UFxcjP3790OtVqOoqAi///57g7Hs7e1RUFCA/Px8lJeXQ61WIyUlpdMtfaVQKODh4YHLly83+npubi6cnJwwd+7cBq+FhYXByckJ586da3YeXePUq4/Bz89Pz+iJiPTD/N5QV8zvLMaJJLR582aMGjUKALB06VKEhoZi69atePvttwEAQ4cOxW+//YYPP/wQUVFRAICJEyfiwoULmjFu3boFPz8/WFhYICgoCAMHDsRXX32l9bzd4sWLMX78eMybNw/e3t544403NH92CwgI0CyTtWjRIjg6OsLX1xeTJ09GcXFxh5yH1pgyZQqysrI068PeTdeasNXV1SgsLERycnKzc+iztuzZs2fh6uqKoUOHNtuXiLoP5vfW63b5Xdxjz549opFmIrrHrFmzxKxZsySNISIiQtjb20saQ0u0Jr9EREQIV1fXBu0XLlwQJiYmYseOHS0ar7a2VgQFBYmEhIQWva8x169fF3K5XKxfv77Ba0qlUvTu3btF47V3/mV+J9IP83vLMb/rpuP8JPLOOJGR0/Ull65CpVLhiy++wIULFzRfqPHy8sKqVauwatUqVFRU6DVObW0t9u/fj/LycoSFhbU5rpiYGDz44IOIjIwEcOdOS0FBAU6dOqXZVIOIqLWY37tHfmcxTkSdXnFxMSZOnIiBAwfimWee0bQvW7YMs2fPRlhYmM4v+9RLTU3Fvn37kJKS0uzObs3ZsGED0tPTcfjwYc3atMnJyXB1dUVQUBAOHTrUpvGJiLoD5ncW41rWrFkDGxsbyGQypKenSx2OXk6fPg0fHx/06NEDMpkMTk5OePPNN6UOS8u+ffvg4eEBmUwGmUwGZ2dnzJ8/X+qwjF50dDS2bduG0tJSuLu7Y+/evVKH1C7ee+89CCE0P5988onW67GxsYiMjMSaNWuaHSs4OBg7d+6Es7Nzm2JKTk7G7du3kZqaCjs7O037tGnTtGK9fv16m+bpjP72t7/BysrKqPIkwPzeXpjf2wfz+x3dJr+34JmWbmHXrl0CgDh//rzUobTI448/LgCImzdvSh1Kkzw9PYWNjY3UYRhMZ3im0Nh09/zSHGN5ZtxY86Sxxs383vGY31uO+V23bvfMuEqlQmBgoNRhdFs8/0TUXphfpMXzT2R4XbIYT0hIQGFhodRhdFs8/0Rdm0wmk2xu5hdp8fwTGZ5BivHa2lq89tprcHNzg4WFBYYOHYo9e/YAALZu3QpLS0soFAokJydj0qRJsLa2Rt++fbFr164GY+3YsQMjR46EXC6HpaUl+vfvjzfeeEPvWJYsWYKoqCjk5eVBJpPBy8sLwJ1vwW7YsAE+Pj4wNzeHnZ0dpk2bhuzsbJ3jXbt2Df3794eJiQkmTpxo8GP++uuvMXr0aCgUClhbW8PPzw9lZWUAgCNHjrR6MX59Y9i0aRPkcjkcHR2xcOFCuLi4QC6XIzAwEGlpaZp+kZGRMDMz03oW6/nnn4elpSVkMpnm2ammzn9LnTx5Er6+vrCxsYFcLoefnx+++OILAHeeWa1/PtHT0xPnz58HADz99NNQKBSwsbHBgQMHAOi+Tm+99RYUCgWsrKxQWFiIqKgouLq6Iicnp1UxE3VFQgisW7cO3t7eMDc3h42NDf71r3816Kfrs1aP+Z35HWB+J2qgBc+0NOmll14S5ubmYu/eveLmzZsiOjpa9OjRQ5w9e1YIIcTy5csFAHHs2DFRWloqCgsLRVBQkLC0tBTV1dWacd5++20BQKxZs0bcuHFDFBcXi/fff1889dRTLYpn5syZwtPTU6vttddeE2ZmZmLHjh2ipKREZGRkiOHDh4v77rtPXL16VdPv3mcKq6urxcyZM0VycrLBj7miokJYW1uLuLg4oVKpxNWrV8WMGTNEUVGREEKIzz//XFhZWYlVq1Y1e8yNPVOo73mPiIgQlpaW4ueffxa3bt0SWVlZYtSoUcLKykr88ccfmn5PPfWUcHJy0pp33bp1AoAm5qbOvxAte6YwKSlJxMTEiOLiYnHjxg3h7++vtabnzJkzRc+ePcWVK1e03vfkk0+KAwcOaP5b3+ukVCrFO++8I2bMmCF++eUXvWLkM4Utx2cKdeuMz4wvX75cyGQy8e9//1vcvHlTVFVViS1btjR49rq5zxrzO/N7Peb3ron5XTddz4y3uRhXqVRCoVCIsLAwTVtVVZUwNzcXixcvFkL83wdCpVJp+tQn89zcXCHEnaRoa2srxo8frzV+TU2NiI+P1zseIRomi6qqKtGrVy+tGIUQ4syZMwKAVjK8O1mr1Woxb948kZKS0i7H/NNPPwkA4vPPP2/R8TVGV7LWFYMQd5L1vUn07NmzAoBYuXKlpq0jk/W9Vq9eLQCIwsJCIYQQR48eFQDEm2++qelTWloqBgwYIGpqaoQQrb9O+mKybjkma906WzFeVVUlFAqFmDBhglb7vUVtc5815ve2YX5nfjcGzO+66SrGTdp6Zz0nJwdVVVUYMmSIps3CwgLOzs46/0RoZmYGAFCr1QCAjIwMlJSU4PHHH9fq17NnTyiVyjbFmJWVhYqKCowcOVKrfdSoUTAzM9P6c1292tpaPPnkk+jTp4/Wny8Bwx2zh4cHHB0dMX/+fCiVSoSHh6N///6tPUy93BtDU0aOHAmFQtHsn3k7Sv06n/UbIDzyyCMYOHAgPvroI0RHR0Mmk2H37t0ICwtDz549AbT+OrXE3r17JX1+1ljxnBmH3NxcVFVVITg4WGe/5j5rzO/M77owv3ctPGct1+ZivLKyEgDw6quv4tVXX9V6zcXFRe9x6p+js7W1bWtIDZSUlAAAevXq1eA1W1tblJeXN2j/xz/+gVu3buHAgQP4+9//Dl9fX81rhjpmCwsLHD9+HK+88gpiY2OxatUqzJkzB9u2bYOFhYXe47QXc3NzFBUVSTL3oUOHsG7dOmRlZaGsrKzB/1xkMhkWLlyIF198EceOHcOjjz6K7du3Y+fOnZo+hrpOuvj7++OFF14wyFjdwXfffYf4+PgGzxLTHfXnp7O4fPkyAMDBwUFnv+Y+a8zvzO93Y37vmpjfddOV39tcjNcn6bfffhtLlixp9Th9+vQBgHbZJKP+fwCNJeWSkhL07du3QfucOXPw17/+FUOGDMGCBQtw+vRpmJjcOV2GOmYAGDx4MA4ePIiioiJs2LABa9euxeDBg7FixYo2jdtWarW6yXPTHk6cOIEffvgBL7zwAv744w9Mnz4dM2bMwEcffYQ+ffrgnXfewcsvv6z1nvDwcERHR+M///kP+vXrB2tra9x///2a1w15nZrSt29fzJkzp13G7qri4+N5znToTMW4XC4HANy+fVtnv+Y+a/VfmmN+Z35nfu/amN91ayq/t3k1lX79+kEul7d5R7P+/fvD3t4eX375ZVtDamDIkCHo1asXvv/+e632tLQ0VFdXY8SIEQ3eM378eNx333344IMP8MMPP2jtemaoYy4oKMDPP/8M4E5iWbNmDYYPH65pk1JqaiqEEPD399e0mZiYNPvnz9b64YcfYGlpCQDIzMyEWq3G4sWL4eHhAblc3uifvezs7DB37lzs378f69evx3PPPaf1uqGuE1F3NWTIEPTo0QNff/21zn7NfdaY35nfmd+JmtbmYlwul+Ppp5/Grl27sHXrVpSVlaG2thaXL1/Gn3/+qfc45ubmiI6OxokTJxAZGYkrV66grq4O5eXlLU5e9vb2KCgoQH5+PsrLy9GzZ09ERUXhs88+wyeffIKysjJkZmZi0aJFcHFxQURERJNjTZ06FeHh4YiNjcUPP/xg0GMuKCjAwoULkZ2djerqapw/fx6///67JkGmpKS0eumrlqqrq8PNmzdRU1ODjIwMLFmyBG5ubggPD9f08fLyQnFxMfbv3w+1Wo2ioiL8/vvvDca69/zrSvBqtRrXrl1DamqqJlm7ubkBAI4ePYpbt27hwoULjT73CQCLFi3C7du38fnnnyMkJETrNUNdJ6LuysHBATNnzsTevXuRkJCAsrIyZGRk4IMPPtDq19xnjfmd+Z35nUiHFnzbs0m3b98WS5cuFW5ubsLExEQ4ODiImTNniqysLLFlyxahUCgEADFgwACRl5cnPvjgA2FtbS0AiPvvv1/8+uuvmrE2b94s/Pz8hFwuF3K5XAwbNkxs2bKlRfGcO3dO3H///cLCwkKMGTNGXL16VdTV1Yl169aJAQMGCFNTU2FnZyemT58ucnJyNO/bt2+fsLOzEwBE//79RWFhoSgrKxP9+vUTAESvXr3E9u3bDXbM+fn5IjAwUNjZ2YmePXuKPn36iOXLl2u+LX748GFhZWWl9Y3ye50+fVoMHjxY9OjRQwAQzs7OIjY2tkXnPSIiQpiamgpXV1dhYmIirK2txbRp00ReXp7WXDdu3BDjx48XcrlcuLu7i3/+85/iX//6lwAgvLy8NMtk3Xv+3333XeHp6SkA6Pz57LPPNHMtXbpU2NvbC1tbWzF79myxefNmAUB4enpqLcclhBDDhg0Ty5Yta/T86LpOcXFxwsLCQgAQ/fr1Ezt27ND3n5gQgt+2bw1+2163zraaihBClJeXi7/97W+id+/eolevXmLMmDHitddeEwBE3759xY8//iiE0P1Zq8f8zvwuBPN7V8X8rpuu1VRkQghxd3GemJiIuXPn4p5m6sIWLlyIpKQk3LhxQ+pQWmXKlCnYvHkz3N3dO3Te2bNnAwCSkpI6dF5jxvyiW3ufH57/7of5vXWY31uO+UU3HecnySA7cJLxq19Syhjc/WfRjIwMyOXyDk/URETGgvmdqHMzimI8Oztbsz2urp+wsDCpQ6UOsHTpUly4cAG//vornn766RZtp03GZ+HChVqf8/nz5zfoc/ToUSxbtgx1dXWYPn063NzcIJfL4erqitDQUGRkZLRozubGOXDgAOLi4hoUOfv379eK9b777mv9gXcTzO90N+b37oX5/Q6jKMYHDRoEIUSzP7t375Y6VKMTHR2Nbdu2obS0FO7u7ti7d6/UITVLoVBg0KBBePTRRxETE6O1RjB1Tfb29khJSUFOTg4SEhK0Xnv99dexadMmREdHo66uDidPnsSnn36K4uJinDp1CiqVCmPHjkVBQYHe8zU3ztSpUyGXyxEcHKxZ5xoAQkNDcfnyZZw4cQKTJ0822PF3Zczv7Yf5nYwB8zsM8wVOou6oM3zBp6qqSgQEBBjNHK3JLxEREcLV1bXR19asWSMGDhyo2e5arVaLJ554QqtP/bbosbGxes+p7ziRkZEiICBAqNXqBmMolUrRu3dvvecUonN+gZOoO2J+bznmd910fYHTKO6ME1HjEhISUFhYaPRztEZubi5WrFiBlStXajanMTExwcGDB7X6eXh4AADy8vL0HlvfcWJiYpCent6pNuohoq6B+b375HcW40QdSAiBDRs2wMfHB+bm5rCzs8O0adOQnZ2t6RMZGQkzMzM4Oztr2p5//nlYWlpCJpNpdjFcsmQJoqKikJeXB5lMBi8vL2zatAlyuRyOjo5YuHAhXFxcIJfLERgYqLWWb1vmAIAjR4502BrJTdm0aROEEJg6darOfiqVCgBgbW3dpvkaG8fOzg7jxo1DfHw8VxAg6uaY3w2nu+V3FuNEHSgmJgbLli3D8uXLUVhYiBMnTuDSpUsICgrCtWvXANxJQvduJ7xlyxasXLlSqy0+Ph4hISHw9PSEEAK5ubmIjIxEeHg4qqqqoFQqkZ+fj3PnzqGmpgYTJkzApUuX2jwH8H+rM9TV1Rnu5LTQoUOH4O3tDYVCobPfmTNnAABjxoxp03xNjTNs2DBcuXIFP/74Y5vGJyLjxvxuON0tv7MYJ+ogKpUKGzZswIwZMzB//nzY2NjAz88P7733Hq5fv95gV8O2MDEx0dyd8fX1xdatW1FeXo5t27YZZPwpU6agrKwMK1asMMh4LVVZWYmLFy/C09OzyT7Xrl3D7t27oVQqERAQ0OwdltaOM2DAAAB3tvkmou6J+d1wumN+N2nX0YlIIysrCxUVFRg5cqRW+6hRo2BmZtbkltCGMHLkSCgUCq0/lxqzwsJCCCF03jUJCAhAZWUl5syZgzfffBOmpqatmqu5cepjqL/zRUTdD/O74XTH/M5inKiD1C+R1KtXrwav2draory8vF3nNzc3R1FRUbvO0VFu3boF4M4xNcXR0REJCQkYPHhwm+ZqbhwLCwutmIio+2F+N5zumN/5mApRB7G1tQWARpNySUkJ+vbt225zq9Xqdp+jI9UnSF07Czo4OGjOeVs0N051dbVWTETU/TC/G053zO+8M07UQYYMGYJevXrh+++/12pPS0tDdXU1RowYoWkzMTHR2ha6rVJTUyGEgL+/f7vN0ZEcHR0hk8lQWlraZJ97l65qrebGqY/BycnJIPMRkfFhfjec7pjfeWecqIPI5XJERUXhs88+wyeffIKysjJkZmZi0aJFcHFxQUREhKavl5cXiouLsX//fqjVahQVFeH3339vMKa9vT0KCgqQn5+P8vJyTfKtq6vDzZs3UVNTg4yMDCxZsgRubm4IDw83yBwpKSmSLn2lUCjg4eGBy5cvN/p6bm4unJycMHfu3AavhYWFwcnJCefOnWt2Hl3j1KuPwc/PT8/oiairYX43nO6Y31mME3Wg119/HatXr8aqVatw3333Ydy4cejfvz9SU1NhaWmp6bd48WKMHz8e8+bNg7e3N9544w3Nn8kCAgI0S1gtWrQIjo6O8PX1xeTJk1FcXAzgzvNtfn5+sLCwQFBQEAYOHIivvvpK6xm8ts4htSlTpiArK0uzPuzddK0JW11djcLCQiQnJzc7hz5ry549exaurq4YOnRos32JqOtifjecbpffW7BdJxHdpTNsl9yYiIgIYW9vL3UYjTLkdskXLlwQJiYmYseOHS0ar7a2VgQFBYmEhIQWva8x169fF3K5XKxfv77BawbeLtkgmN+J9MP83nLM77rpOD+JvDNO1AXp+uKLMVKpVPjiiy9w4cIFzRdqvLy8sGrVKqxatQoVFRV6jVNbW4v9+/ejvLwcYWFhbY4rJiYGDz74ICIjIwHcudNSUFCAU6dOaTbQICIyJOb3xhlzfmcxTkSdXnFxMSZOnIiBAwfimWee0bQvW7YMs2fPRlhYmM4v+9RLTU3Fvn37kJKS0uzObs3ZsGED0tPTcfjwYc3atMnJyXB1dUVQUBAOHTrUpvGJiLoD5ncW40RdSnR0NLZt24bS0lK4u7tj7969UofUZu+99x6EEJqfTz75ROv12NhYREZGYs2aNc2OFRwcjJ07d8LZ2blNMSUnJ+P27dtITU2FnZ2dpn3atGlasV6/fr1N8xAR1WN+182Y8zuXNiTqQlavXo3Vq1dLHUaHe+yxx/DYY4912HyhoaEIDQ3tsPmIiJjfO4YU+Z13xomIiIiIJMJinIiIiIhIIizGiYiIiIgkwmKciIiIiEgiTX6BMzExsSPjIDI69dvkdsRnRQgBmUzW7vO0t++++w4A80tT6s9Pe+P57x66St6QQkfm966C+V03XfldJoT2fqCJiYmYO3duuwdFRESNE3ps09wazO9ERNJqJL8nNSjGiahz+eWXXxAZGYljx45h/vz5iIuLg4uLi9RhEVEndOLECfzzn/9EdnY2lixZghUrVqBXr15Sh0VETUviM+NEnZyPjw/+97//ITk5GadOncKAAQMQExOD27dvSx0aEXUSBQUFWLBgAf7yl7/AwcEB586dQ1xcHAtxIiPAYpzISISEhCArKwsrVqzAv//9b/j5+XHLdaJuTq1WY+PGjRg0aBBSU1Px8ccf4+jRoxg8eLDUoRGRnliMExkRCwsLLF26FL/88gv8/f3xxBNPICQkBL/99pvUoRFRBzt69CgeeOABREdH48UXX8Svv/6KBQsWSB0WEbUQi3EiI9S3b19s374dx48fR35+Pnx8fKBUKlFeXi51aETUznJzcxESEoIJEybA09MTWVlZiImJgVwulzo0ImoFFuNERmz8+PE4f/483nrrLfz3v/+Fj48Ptm/f3m6rcRCRdKqqqhATE4MhQ4YgNzcXR44cwcGDB9G/f3+pQyOiNmAxTmTkTExMoFQqkZeXh5kzZ+Lpp5/G+PHjkZGRIXVoRGQgBw8ehI+PDzZu3Ii4uDhkZmbi8ccflzosIjIAFuNEXUTv3r2xceNGnDlzBtXV1RgxYgQiIiJw/fp1qUMjolZKT0/H2LFjERoainHjxiEnJwdKpRImJk3u2UdERobFOFEXM2LECHzzzTdISEhAcnIyvL29sXHjRtTW1kodGhHp6ebNm1AqlRg5ciRUKhW+/fZbbN++HY6OjlKHRkQGxmKcqAuSyWRYsGABsrOz8dxzz+Hll1/G6NGj8c0330gdGhHpUFdXh+3bt8Pb2xuJiYnYunUr0tLS4O/vL3VoRNROWIwTdWG2trZYu3YtMjIy4ODggKCgIMyZMweXLl2SOjQiuseZM2cQEBCAv/3tb5g3bx6ys7Px97//HT168H/VRF0ZP+FE3YC3tzeOHDmC5ORknD17Fj4+PtzFk6iTqN8909/fH7169cL58+exceNG2NjYSB0aEXUAFuNE3UhISAh+/vlnvPTSS4iLi4Ofnx8OHz4sdVhE3VL97pk+Pj746quv8PHHH+PYsWPcPZOom2ExTtTNWFhYICYmBhcuXIC/vz+mTJnCXTyJOtjRo0fx4IMPYtmyZXjhhRdw4cIF7p5J1E2xGCfqpup38Tx27BguXryo2cWzoqJC6tCIuqy8vDzMmTMHEyZMgIeHB37++WfunknUzbEYJ+rmHnnkEa1dPAcNGoTt27dLHRZRl3L37pkZGRncPZOINFiMExFMTU2hVCqRnZ2NKVOmaHbxzMzMlDo0IqN38OBB+Pr6YuPGjYiJiUFGRgZ3zyQiDRbjRKTh7OyM999/H2lpabh16xaGD/2fgz8AACAASURBVB+OiIgI3LhxQ+rQiIxOdnY2HnvsMYSGhmLs2LHIzs7G0qVLYWZmJnVoRNSJsBgnogZGjhyJb7/9FgkJCdi/fz8GDhzIXTyJ9FS/e6afnx+Ki4vxzTffYPv27XBycpI6NCLqhFiME1Gj6nfxzMnJwfz58/HSSy9h9OjR+Pbbb6UOjahTqt89c9CgQdi5cyfWr1+v2ciHiKgpLMaJSCdbW1ts3LgRmZmZcHBwwJgxY7BgwQJcvXpV6tCIOo2zZ88iMDAQzz77LMLCwpCXlwelUsndM4moWcwSRKSXQYMGaXbxPHnyJLy8vBATE4Pq6mqpQyOSzJ9//okFCxbgoYcegkKh4O6ZRNRiLMaJqEVCQkKQlZWltYtnSkqK1GERdaj63TMHDRqk2T3z+PHjGDJkiNShEZGRYTFORC2mUCgQExODX3/9FQ899BAmT56MkJAQXLx4UerQiNrdsWPHMGzYMCxbtgyLFi3CL7/8wt0ziajVWIwTUav169cP27dvx9GjR3Hx4kUMHjwYr7zyCnfxpC6pfvfMRx99FO7u7sjKysLatWvRq1cvqUMjIiPGYpyI2iw4OBjnz5/HmjVr8O6778LHx4e7eFKXcffumT/++CNSUlJw8OBBuLu7Sx0aEXUBLMaJyCDu3sVz8uTJePrpp/HII4/gp59+kjo0olar3z1z/fr1WLp0KTIzMzFx4kSpwyKiLoTFOBEZlIuLC95//32cPn0aVVVVGDZsGJRKJUpLS6UOjUhv2dnZePzxxzW7Z+bl5SEmJoa7ZxKRwbEYJ6J2MWrUKM0unrt374anpyc2btyIuro6qUMjalJJSYlm98zr16/j1KlT3D2TiNoVi3Eiajc9evTAggULkJ2djaeeekqzi+d3330ndWhEWup3z/T29tbaPTMwMFDq0Iioi2MxTkTtzs7OTrOLZ+/evfHwww9jwYIFuHbtmtShEeH777/Hww8/jGeffRbTpk1DTk4OlEolevbsKXVoRNQNsBgnog4zaNAgfPHFF0hOTsaJEycwaNAgxMXFcRdPksTVq1cRERGBhx56CHK5HOfOncP777+P3r17Sx0aEXUjLMaJqMOFhITg559/hlKpRExMDIYOHYojR45IHRZ1E/W7Z3p7e+PQoUPYtm0bvvrqK/j5+UkdGhF1QyzGiUgS9bt4/vTTTxg6dCgmTZqEkJAQ5OfnSx0adWHHjx/X2j0zOzubu2cSkaRYjBORpDw9PZGYmIijR4/it99+g6+vL2JiYqBSqaQOjbqQS5cuYcGCBQgODubumUTUqbAYJ6JOITg4GOnp6VizZg3efvttDBw4kLt4UpupVCrExMRg4MCBSEtLw+HDh7l7JhF1KizGiajTuHsXz/HjxyM8PBzBwcF67eL5+eefd0CE1BmoVCocP3682X737p6ZkZGBSZMmdUCERET6YzFORJ2Oi4sLtm/fjrS0NFRUVGD48OE6d/E8ePAgpk6dij179nRwpNTRhBB46qmnMHfu3Cb/PeTk5GDixIkIDQ1FUFAQcnNzERMTA3Nz8w6OloioeSzGiajTGjVqFL777jv85z//wa5duxrdxfP27dv4xz/+AQD461//itOnT0sVLnWAV199FcnJySgpKcHKlSu1XispKcErr7yCoUOHoqioCCdPnsT27dvh7OwsUbRERM1jMU5EnVr9Lp45OTl46qmnEBUVhYceekhTdK9fvx5XrlyBEAJ1dXWYNGkS8vLyJI6a2sOePXuwZs0a1NXVoaamBps2bUJmZiaEENi+fTsGDRqEDz/8EG+99RbOnDmDhx9+WOqQiYiaJRNCCKmDICLSV3p6OiIjI/HNN99g3rx52LdvH27duqV53dTUFP3798fZs2dhY2MjYaRkSKdOncIjjzwCtVqtaTM1NcWwYcMghMD58+exePFirFy5Era2thJGSkTUIkm8M05ERuXBBx/E119/jR07duD7779HbW2t1utqtRr5+fmYPXs2ampqJIqSDOnixYsIDQ3VejwJuHOtz5w5A1NTU5w/fx4bN25kIU5ERofFOBEZHZlMBjc3N/z6669ad0rrqdVqHD9+HFFRURJER4ZUVlaGSZMmoaysrMEvXsCdx5guXrwIDw8PCaIjImo7FuNEZHRqa2sRERGBHj2aTmG1tbV45513sGXLlg6MjAyppqYG06ZNw2+//dbkXznq6upQVFSEdevWdXB0RESGwWKciIzOu+++i+zs7EbvlN5NCIHIyEh8+eWXHRQZGdI//vEPnDhxotG/ftytpqYGq1evRn5+fscERkRkQCzGicioXL9+HStWrAAAmJmZ6fWeGTNm4Oeff27PsMjA4uPj8cEHHzT7C5dMJoOZmRmqq6v5WBIRGSWupkJERqW2thbZ2dn44Ycf8MMPPyAtLQ0//vgjbt26hR49esDU1BS3b9/Weo+pqSkcHR1x7tw5ODo6ShQ56SslJQVPPPFEgy9s3l14CyFgZWWFwYMHY/To0RgxYgRGjBiBwYMHSxQ1EVGrJLEYJyKjV1tbi5ycHPzwww84d+6cpkCvqqpCjx490LNnT6jVaowePRonTpzgToydWEZGBvz9/aFSqWBqaqp5RMXBwQGjR4/G6NGjMXz4cAwfPhx9+vSROFoiojZjMU6di0wmkzoEIiLqwmbNmoWkpCSpwyCql2QidQRE91qyZAkCAgKkDoO6ICEErl69CltbW1hYWEgdTrubO3euUX2eioqKIJfLYWVlJVkMb7/9NgDghRdekCwGaj/115eoM2ExTp1OQEAA5syZI3UYREZv7ty5/Dy1UP0dU56zrol3xKkz4moqREREREQSYTFORERERCQRFuNERERERBJhMU5EREREJBEW40REREREEmExTkREOh0+fBg2NjY4ePCg1KF0ekePHsWyZcuabK+rq8P06dPh5uYGuVwOV1dXhIaGIiMjo0XzNDfOgQMHEBcXh9raWoMel9TxG/q4iDoDFuNERKQT94bTz+uvv45NmzYhOjq6yfa6ujqcPHkSn376KYqLi3Hq1CmoVCqMHTsWBQUFes/V3DhTp06FXC5HcHAwSkpKDHZcUsdvyOMi6ixYjBMRkU5TpkxBaWkpQkJCpA4FKpUKgYGBUofRwNq1a7F7924kJiZqbVrUWHtAQADGjBkDhUIBd3d3xMbGorS0FB9//HGL5mxuHKVSiQceeACTJ09GTU2NwY5L6vgNcVxEnQmLcSIiMhoJCQkoLCyUOgwtubm5WLFiBVauXAm5XK6z3cTEpMHjPh4eHgCAvLw8vefUd5yYmBikp6cjPj6+ZQfVyeNvy3ERdTYsxomIqEmnTp2Cm5sbZDIZNm/eDADYunUrLC0toVAokJycjEmTJsHa2hp9+/bFrl27NO/dtGkT5HI5HB0dsXDhQri4uEAulyMwMBBpaWmafpGRkTAzM4Ozs7Om7fnnn4elpSVkMhmuX78OAFiyZAmioqKQl5cHmUwGLy8vAMCRI0dgbW2N2NjYjjglDWzatAlCCEydOlWv9nupVCoAgLW1dZviaGwcOzs7jBs3DvHx8S1+3Kgzx9+W4yLqbFiMExFRk8aMGYNvv/1Wq23x4sV44YUXoFKpYGVlhT179iAvLw8eHh547rnnoFarAdwpssPDw1FVVQWlUon8/HycO3cONTU1mDBhAi5dugTgTtF37/bzW7ZswcqVK7Xa4uPjERISAk9PTwghkJubCwCaL/PV1dW1yzlozqFDh+Dt7Q2FQqFX+73OnDkD4M65boumxhk2bBiuXLmCH3/8sUXjdfb4W3tcRJ0Ni3EiImq1wMBAWFtbw8HBAWFhYaisrMQff/yh1cfExAQ+Pj4wNzeHr68vtm7divLycmzbts0gMUyZMgVlZWVYsWKFQcZricrKSly8eBGenp56td/t2rVr2L17N5RKJQICApq9A93acQYMGAAAyMzM1HtMY4i/NcdF1BmZSB0AERF1DWZmZgCguTPelJEjR0KhUCA7O7sjwmpXhYWFEEI0uHvcVPvdAgICUFlZiTlz5uDNN9+Eqalpq2Jobpz6GK5du6b3mMYQf2uOi6gzYjFOREQdztzcHEVFRVKH0Wa3bt0CcOd49Gm/m6OjIxISEjB48OA2xdDcOBYWFlox6cMY4m/NcRF1RnxMhYiIOpRarUZJSQn69u0rdShtVl8Q3rsJTVPtd3NwcICtrW2bY2hunOrqaq2Y9GEM8bfmuIg6I94ZJyKiDpWamgohBPz9/TVtJiYmzT7e0hk5OjpCJpOhtLRUr/a7GWpH0+bGqY/ByclJ7zGNIf7WHBdRZ8Q740RE1K7q6upw8+ZN1NTUICMjA0uWLIGbmxvCw8M1fby8vFBcXIz9+/dDrVajqKgIv//+e4Ox7O3tUVBQgPz8fJSXl0OtViMlJUWypQ0VCgU8PDxw+fJlvdrr5ebmwsnJCXPnzm3wWlhYGJycnHDu3Llm59c1Tr36GPz8/PQevzPH31w7kbFhMU5ERE3avHkzRo0aBQBYunQpQkNDsXXrVrz99tsAgKFDh+K3337Dhx9+iKioKADAxIkTceHCBc0Yt27dgp+fHywsLBAUFISBAwfiq6++0noeefHixRg/fjzmzZsHb29vvPHGG5rHDwICAjTLIC5atAiOjo7w9fXF5MmTUVxc3CHnQZcpU6YgKytLs052c+0AdK6NXV1djcLCQiQnJzc7tz5rbJ89exaurq4YOnRoi8bvrPE3105kdARRJwJA7NmzR+owiLqEzvB5ioiIEPb29pLG0BKzZs0Ss2bNatF7Lly4IExMTMSOHTv0am9ObW2tCAoKEgkJCS16X2OuX78u5HK5WL9+fYvH76zx62pvTmuuL1E7S+SdcSIiale6vgTYFXh5eWHVqlVYtWoVKioqmm3Xpba2Fvv370d5eTnCwsLaHFtMTAwefPBBREZGtnj8zhh/c+1ExojFOJGB3b59G0qlEs7OzlAoFDhy5IjUIbWbsLAwyGQyvX4+//xzqcPtVteGOtayZcswe/ZshIWFaX3psan2pqSmpmLfvn1ISUlpdufL5mzYsAHp6ek4fPiwZu3ulo7f2eLX1U5krFiMExnYv//9bxw5cgTZ2dmIj4/X+46Ssfryyy9RUlICtVqNP//8EwAwdepUVFdXo7KyEoWFhXjuueckjvKO7nZtpBYdHY1t27ahtLQU7u7u2Lt3r9QhtavY2FhERkZizZo1erU3Jjg4GDt37oSzs3ObYklOTsbt27eRmpoKOzu7No3fmeJvqp3ImMmE0OPbE0QdRCaTYc+ePZgzZ45e/VUqFYKDg/Htt9+2c2T6Gz16NAYMGICdO3dKHUq7mzdvHhISEjR3wK5evQoXFxeEhoZi//79mn7vv/8+XF1d8cQTT0gVKoDudW2Aln+eCJg9ezYAICkpSeJIqD3w+lInlMR1xsmoJSQkoLCwUOowtFy+fBm+vr5Sh9Ehdu3apVe/iIiIdo5EP93p2hARkXHgYypktJYsWYKoqCjk5eVBJpPBy8sLb731FhQKBaysrFBYWIioqCi4uroiJycHJ0+ehK+vL2xsbCCXy+Hn54cvvvgCALB161ZYWlpCoVAgOTkZkyZNgrW1Nfr27dug4Pz6668xevRoKBQKWFtbw8/PD2VlZfjf//4HLy8v/Pnnn/jvf/8LmUyGXr16AbizfNeGDRvg4+MDc3Nz2NnZYdq0acjOztaM21TsixYtgqWlJXr06IERI0bAyckJpqamsLS0xPDhwxEUFIR+/fpBLpfD1tYWL7/8sla8tbW1eO211+Dm5gYLCwsMHToUe/bs0TlnTk4Ojhw5YtC1m3ltDHdtiIioC5F2NRcibWjhUmwzZ84Unp6eWm3Lly8XAIRSqRTvvPOOmDFjhvjll19EUlKSiImJEcXFxeLGjRvC399f9O7du8H7jh07JkpLS0VhYaEICgoSlpaWorq6WgghREVFhbC2thZxcXFCpVKJq1evihkzZoiioiLNOE5OTuL//b//pxXTa6+9JszMzMSOHTtESUmJyMjIEMOHDxf33XefuHr1arOxv/766wKASEtLE5WVleL69eti4sSJAoA4dOiQKCoqEpWVlSIyMlIAEOnp6ZoxX3rpJWFubi727t0rbt68KaKjo0WPHj3E2bNndc75+eefCysrK7Fq1Sq9r8eff/4pAIjQ0NBGX+e1Mcy10VdLP0/Epe+6Ol5f6oQSWYxTp2LIYlylUul87+rVqwUAUVhY2OT7tmzZIgCI3NxcIYQQP/30kwAgPv/88ybHvbfgq6qqEr169RJhYWFa/c6cOSMAaBW7TcVeX/CVl5dr2v773/8KACIzM7PBmLt37xZCCKFSqYRCodCau6qqSpibm4vFixe36HzpQ99inNemY64Ni/GWY7HWtfH6UieUyGfGqduqXxJL1xrIZmZmAAC1Wg0A8PDwgKOjI+bPnw+lUonw8HD0799f5zxZWVmoqKjAyJEjtdpHjRoFMzMzpKWltSr++thqamo0bfXHVB9vTk4OqqqqMGTIEE0fCwsLODs7az2G0dnw2hju2nz33XcGG6s7qN9iPTExUeJIqD1cvnwZffv2lToMIi0sxqnbOHToENatW4esrCyUlZVpiqKWsLCwwPHjx/HKK68gNjYWq1atwpw5c7Bt2zbN1t33KikpAQDNM8p3s7W1RXl5eYvj0FdlZSUA4NVXX8Wrr76q9ZqLi0u7zdtSvDbtd23i4+MRHx9vsPG6i7lz50odArWTWbNmSR0CkRZ+gZO6hT/++APTp0+Hs7Mz0tLSUFpairi4uFaNNXjwYBw8eBAFBQVYunQp9uzZg/Xr1zfZ39bWFgAaLexKSkra9S6Ng4MDAODtt9+GEELrp7PcMeW1ad9rs2fPngbj86fpn1mzZmHWrFmSx8Gf9ru+RJ0Ni3HqFjIzM6FWq7F48WJ4eHhALpdDJpO1eJyCggL8/PPPAO4UU2vWrMHw4cM1bY0ZMmQIevXqhe+//16rPS0tDdXV1RgxYkSL49BX/Uoe6enp7TZHW/HadN5rQ0RE7Y/FOBk1e3t7FBQUID8/H+Xl5U0+3uDm5gYAOHr0KG7duoULFy606nnggoICLFy4ENnZ2aiursb58+fx+++/w9/fv8n3yOVyREVF4bPPPsMnn3yCsrIyZGZmYtGiRXBxcWnXNbjlcjmefvpp7Nq1C1u3bkVZWRlqa2tx+fJlzW6ZTUlJSTHo0oZN4bVp+bUhIqIuRBB1Imjh6g/nzp0T999/v7CwsBBjxowRL774orCwsBAARL9+/cSOHTs0fZcuXSrs7e2Fra2tmD17tti8ebMAIDw9PcUrr7wiFAqFACAGDBgg8vLyxAcffCCsra0FAHH//feLX3/9VeTn54vAwEBhZ2cnevbsKfr06SOWL18uampqRH5+vhg2bJgAIExMTMTw4cPF3r17hRBC1NXViXXr1okBAwYIU1NTYWdnJ6ZPny5ycnI08cXFxTUae3x8vCa2/v37i5MnT4q1a9cKGxsbAUA4OTmJnTt3it27dwsnJycBQNjZ2Yldu3YJIYS4ffu2WLp0qXBzcxMmJibCwcFBzJw5U2RlZTU5pxBCHD58WFhZWYk333yz2etQVlYmxo4dK+zt7QUA0aNHD+Hl5SViY2ObPT5em5ZfG3219PNEXG2jq+P1pU4oUSaEEB1Y+xPpxO27iQyHn6eW43bpXRuvL3VCSXxMhYiIiIhIIizGiYiIiIgkwmKciIionRw9ehTLli1DXV0dpk+fDjc3N8jlcri6uiI0NBQZGRktGs9Q49T79NNPMWrUKFhZWeH+++/H008/jatXrzbod+rUKTz88MNQKBRwcXHB0qVLcfv27Vb1+8tf/gKZTNboT/2a/wcOHEBcXJzOjb+IugoW40RERO3g9ddfx6ZNmxAdHY26ujqcPHkSn376KYqLi3Hq1CmoVCqMHTsWBQUFeo9pqHGAO2vQP/XUU5g9ezYuX76M5ORknDhxApMmTdLaPTYrKwuPPfYYgoODUVRUhM8++wwfffQRFi1apDWevv10GTNmDABg6tSpkMvlCA4O1mzORdRVsRgnIqJ2oVKpEBgYaPRztMbatWuxe/duJCYmwsrKCgAQEBCAMWPGQKFQwN3dHbGxsSgtLcXHH3/corENNc7777+PPn364F//+hdsbGzw4IMP4sUXX0R6errW8qJvvPEGnJ2dsXLlSlhaWiIgIABLly7Fxx9/jOzs7Bb3k8vlKCsra7AhT0REBF5++WVNP6VSiQceeACTJ0/W+uWAqKthMU5ERO0iISEBhYWFRj9HS+Xm5mLFihVYuXIl5HI5AMDExAQHDx7U6ufh4QEAyMvL03tsQ40DAJcuXYKLi4vWJlv9+vUDAPz+++8AgJqaGhw6dAjjxo3T6jdp0iQIIZCcnNyifgBw5MgRzS8od8fy008/4ZFHHtFqj4mJQXp6OuLj41t0bETGhMU4EREBAIQQ2LBhA3x8fGBubg47OztMmzZN665mZGQkzMzM4OzsrGl7/vnnYWlpCZlMhuvXrwMAlixZgqioKOTl5UEmk8HLywubNm2CXC6Ho6MjFi5cCBcXF8jlcgQGBmrdiW3LHMCdYq8jNqxqyqZNmyCEwNSpU3X2U6lUAABra+s2zdfacTw8PBr8IlP/vHh9gf/bb7+hoqJCszlXPU9PTwDQPKuub7+mrF27FkqlskG7nZ0dxo0bh/j4eHAlZuqqWIwTERGAO3chly1bhuXLl6OwsBAnTpzApUuXEBQUhGvXrgG4U2jeu275li1bsHLlSq22+Ph4hISEwNPTE0II5ObmIjIyEuHh4aiqqoJSqUR+fj7OnTuHmpoaTJgwAZcuXWrzHAA0X/qrq6sz3MlpgUOHDsHb2xsKhUJnvzNnzgD4v+ekW6u140RHR+Pq1at45513UF5ejqysLMTHx+Pxxx/X7FxbX5zfeydbLpfDwsJC8+9C336NuXLlClJTUzFz5sxGXx82bBiuXLmCH3/8sUXHR2QsWIwTERFUKhU2bNiAGTNmYP78+bCxsYGfnx/ee+89XL9+HR988IHB5jIxMdHcfff19cXWrVtRXl6Obdu2GWT8KVOmoKysDCtWrDDIeC1RWVmJixcvau4IN+batWvYvXs3lEolAgICmr2D3l7jjBs3DkuXLkVkZCSsra0xZMgQlJeX4z//+Y+mT/1KKD179mzwflNTU81deX37NWbt2rX45z//iR49Gi9JBgwYAADIzMzU88iIjAuLcSIiQlZWFioqKjBy5Eit9lGjRsHMzEzrMRJDGzlyJBQKhdbjMMaqsLAQQgidd8UDAgKgVCoxbdo0pKSkwNTUtFVztXWc5cuX44MPPsCxY8dQUVGB3377DYGBgQgICND8laL+mffGvkBZXV0NCwuLFvW7V0FBAQ4cOIDw8PAm46w/l7rurhMZMxbjRESkWT6ufp3nu9na2qK8vLxd5zc3N0dRUVG7ztERbt26BeDO8TTF0dERx48fxzvvvAMbG5tWz9WWcf7880/ExcXh73//Ox555BFYWlrC3d0dH374IQoKCrBu3ToA0Dy3X1ZWpvX+qqoq3Lp1Cy4uLi3qd6+4uDg899xzmmK+MfWFfP25JepqTKQOgIiIpGdrawsAjRbdJSUl6Nu3b7vNrVar232OjlJfOOrarMbBwUFzvtuiLeNcuHABtbW16NOnj1a7tbU17O3tkZWVBQBwd3eHlZWVZnWVevXP5w8dOrRF/e529epVfPrpp8jJydEZa3V1NQA0eXedyNixGCciIgwZMgS9evXC999/r9WelpaG6upqjBgxQtNmYmICtVptsLlTU1MhhNB8abA95ugojo6OkMlkKC0tbbLPvUsTtlZbxqn/xefPP//Uai8vL0dxcbFmiUMTExNMnjwZJ06cQF1dnea57pSUFMhkMs1z6vr2u1tcXBzmz58Pe3t7nbHWn0snJ6dWHy9RZ8bHVIiICHK5HFFRUfjss8/wySefoKysDJmZmVi0aBFcXFwQERGh6evl5YXi4mLs378farUaRUVFDe6IAoC9vT0KCgqQn5+P8vJyTXFdV1eHmzdvoqamBhkZGViyZAnc3Ny0nhtuyxwpKSmSLW2oUCjg4eGBy5cvN/p6bm4unJycMHfu3AavhYWFwcnJCefOnWt2nraO4+7ujvHjx+PDDz/EiRMnoFKpcOnSJc11fvbZZzV9V6xYgWvXruH1119HZWUlvvvuO6xbtw7h4eHw9vZucT/gzvPfH330EV544YVmj7X+XPr5+TXbl8gYsRgnIiIAd7ZvX716NVatWoX77rsP48aNQ//+/ZGamgpLS0tNv8WLF2P8+PGYN28evL298cYbb2geIbj7y3+LFi2Co6MjfH19MXnyZBQXFwO48+yvn58fLCwsEBQUhIEDB+Krr77Ses66rXNIacqUKcjKymp0BRFda2VXV1ejsLBQa4OcprR1HJlMhqSkJISFheHZZ5+FnZ0dfH198ccff2Dfvn0ICgrS9B08eDC++OILfPnll+jduzdmzpyJZ555Bu+++67WmPr2A4C33noLU6dObbAueWPOnj0LV1fXRh91IeoKZIKr6FMnIpPJsGfPngZrDBNRy3XGz9PChQuRlJSEGzduSB1Ko2bPng0ASEpKavUYubm58PHxwbZt2zB//ny931dXV4e//OUvCA8PxzPPPNPq+Q01Tmdw48YN9O3bF2+++SaioqLaPJ4hri+RgSXxzjgREXUoXV9u7Aq8vLywatUqrFq1ChUVFXq9p7a2Fvv370d5eTnCwsJaPbehxuksYmJi8OCDDyIyMlLqUIjaDYtxIiIiA1u2bBlmz56NsLAwnV/mrJeamop9+/YhJSWl2Z07O2KczmDDhg1IT0/H4cOHW70WO5ExYDFOREQdIjo6Gtu2bUNpaSnc3d2xd+9eqUNqV7GxsYiMjMSaNWua7RscHIydO3dq1utu7KJsngAAHxlJREFULUONI7Xk5GTcvn0bqampsLOzkzoconbFpQ2JiKhDrF69GqtXr5Y6jA712GOP4bHHHpM6DKMTGhqK0NBQqcMg6hC8M05EREREJBEW40REREREEmExTkREREQkERbjREREREQS4aY/1KnIZDL4+/ujb9++UodCZPT27t3Lz1MLnT59GgDg7+8vcSTUHk6fPg1/f39u+kOdSRKLcepU6ndHI6LmXb16FefPn8ekSZOkDoXIaAQEBODFF1+UOgyieizGiYiMVWJiIubOnQumcSIio5XEZ8aJiIiIiCTCYpyIiIiISCIsxomIiIiIJMJinIiIiIhIIizGiYiIiIgkwmKciIiIiEgiLMaJiIiIiCTCYpyIiIiISCIsxomIiIiIJMJinIiIiIhIIizGiYiIiIgkwmKciIiIiEgiLMaJiIiIiCTCYpyIiIiISCIsxomIiIiIJMJinIiIiIhIIizGiYiIiIgkwmKciIiIiEgiLMaJiIiIiCTCYpyIiIiISCIsxomIiIiIJMJinIiIiIhIIizGiYiIiIgkwmKciIiIiEgiLMaJiIiIiCTCYpyIiIiISCIsxomIiIiIJMJinIiIiIhIIizGiYiIiIgkwmKciIiIiEgiLMaJiIiIiCTCYpyIiIiISCImUgdARETNU6vVqKio0GqrrKwEANy8eVOrXSaTwdbWtsNiIyKi1mMxTkRkBIqLi+Hq6ora2toGr9nb22v99/jx43H8+PGOCo2IiNqAj6kQERkBJycnjB07Fj166E7bMpkM8+bN66CoiIiorViMExEZib/+9a/N9unZsydmzJjRAdEQEZEhsBgnIjISM2fOhIlJ008X9uzZExMnTkTv3r07MCoiImoLFuNEREbC2toakyZNarIgF0Jg/vz5HRwVERG1BYtxIiIjMn/+/Ea/xAkAZmZmeOKJJzo4IiIiagsW40RERuSJJ56AQqFo0G5qaorp06fD0tJSgqiIiKi1WIwTERkRuVyOGTNmwNTUVKtdrVbjqaeekigqIiJqLRbjRERG5sknn4RardZqs7a2xoQJEySKiIiIWovFOBGRkXn00Ue1NvoxNTXFvHnzYGZmJmFURETUGizGiYiMjImJCebNm6d5VEWtVuPJJ5+UOCoiImoNFuNEREZo3rx5mkdVnJycMGbMGIkjIiKi1mAxTkRkhAIDA+Hq6goAWLBgAXr0YDonIjJGTW/lRkRtlpiYKHUI1IWNGjUKV65cQe/evflvjdpNv379EBAQIHUYRF2WTAghpA6CqKuSyWRSh0BE1CazZs1CUlKS1GEQdVVJvDNO1M727NmDOXPmSB0GGcjs2bMBoNMUJ3v37sWsWbOkDkOnxMREzJ07F7z3Y3zq/73T/2/v3oOqLvMHjr+PcTkc7pQgXjAuyUJSVjgrJKGyUeoG3gha3Yl0vGAT6LLFosOA1yCKWDVttbRmkkTSBRvFrrLoWpZjXqLVEY1EkYuScFCunu/vD3/nrCcQuR9wP68Z/3n4fD/P8/3CyOc8PN/nEaL3yCJDIYQYwPp7IS6EEKJ9UowLIYQQQghhIlKMCyGEEEIIYSJSjAshhBBCCGEiUowLIYQQQghhIlKMCyGEEEIIYSJSjAshhAns27cPe3t7Pv30U1MPpd/78ssvSUxMRKfTMX36dNzc3FCr1QwbNozw8HBOnjzZqXw9lUcvKyuLsWPHYmtry8iRI3nppZcoLy9vFXfo0CGefPJJNBoNrq6uJCQk0NjY2KW4CRMmoFKp2vxnY2MDwJ49e0hLS+PmzZtdui8hRN+QYlwIIUxA9tzumOTkZNatW8eyZcvQ6XQcPHiQrKwsqqurOXToEPX19Tz11FOUlZV1OGdP5YFb5wjMnj2biIgILl68SF5eHoWFhUyePJmWlhZDXFFREaGhoYSEhFBVVcXu3bvZunUrMTExRvk6Gtee8ePHAxAWFoZarSYkJIRr16516r6EEH1IEUL0GkDJzs429TBED5o1a5Yya9YsUw+jR924cUMJCAjotfzZ2dlKV37dvP7668qoUaOU+vp6RVEUpbm5WfnjH/9oFPPdd98pgLJmzZoO5+2pPIqiKBMnTlSGDh2q6HQ6Q9uGDRsUQDl06JChLTIyUnF3dzeKS09PV1QqlfKf//yn03HPPPOMUltb22o8CxcuVL766iujttjYWCUgIEBpbm7u1L0pyr358y5EP7NTZsaFEOJ/3Pvvv09lZaWph2GkuLiYpKQkVqxYgVqtBsDMzKzVsh4PDw8Azp071+HcPZUHoLS0FFdXV1QqlaFtxIgRAPzyyy8AtLS0sHfvXoKDg43iJk+ejKIo5OXldSoOYP/+/dja2rYay48//sikSZOM2lNSUjh+/DiZmZmdujchRN+QYlwIIfrYoUOHcHNzQ6VSsWHDBgA2btyItbU1Go2GvLw8Jk+ejJ2dHcOHD+fjjz82XLtu3TrUajXOzs4sWrQIV1dX1Go1gYGBHDlyxBAXGxuLhYUFQ4YMMbS9/PLLWFtbo1KpuHLlCgBLliwhPj6ec+fOoVKp8PLyAm4Ve3Z2dqxZs6YvHkkr69atQ1EUwsLC2o2rr68HwM7Orlv9dTWPh4dHqw8y+vXi+gL//Pnz1NXV4ebmZhTn6ekJYFir3tG4O0lNTSUuLq5Vu6OjI8HBwWRmZsryKCH6ISnGhRCij40fP57Dhw8btS1evJilS5dSX1+Pra0t2dnZnDt3Dg8PD+bPn09zczNwq8iOjo7mxo0bxMXFUVJSwrFjx2hpaeHpp5+mtLQUuFXMPv/880Z9vPPOO6xYscKoLTMzk+eeew5PT08URaG4uBjA8NKfTqfrlWdwN3v37sXb2xuNRtNu3HfffQf8d510V3U1z7JlyygvL2f9+vVotVqKiorIzMzkmWeeYdy4ccB/i/PfzmSr1WqsrKyoqKjoVFxbLl26REFBATNnzmzz64899hiXLl3ixIkTnbo/IUTvk2JcCCH6mcDAQOzs7Bg8eDBRUVFcv36dCxcuGMWYmZnh4+ODpaUlvr6+bNy4Ea1Wy7Zt23pkDFOnTqW2tpakpKQeydcZ169f5+effzbMCLeloqKCHTt2EBcXR0BAwF1n0HsrT3BwMAkJCcTGxmJnZ8fo0aPRarW89957hhj9Tij33Xdfq+vNzc0Ns/IdjWtLamoqr7zyCoMGtf1r/aGHHgLg1KlTHbwzIURfkWJcCCH6MQsLCwDDzPid+Pv7o9FoOH36dF8Mq1dVVlaiKEq7s+IBAQHExcUxbdo08vPzMTc371Jf3c2zfPlyNm/ezFdffUVdXR3nz58nMDCQgIAAw18p9Gveb99dRa+pqQkrK6tOxf1WWVkZe/bsITo6+o7j1D/L9mbXhRCmIcW4EELcIywtLamqqjL1MLqtoaEBuHU/d+Ls7MzXX3/N+vXrsbe373Jf3clz+fJl0tLSWLBgAZMmTcLa2hp3d3e2bNlCWVkZ6enpAIZ1+7W1tUbX37hxg4aGBlxdXTsV91tpaWnMnz/fUMy3RV/I65+tEKL/MDP1AIQQQnRfc3Mz165dY/jw4aYeSrfpC8f2DqsZPHgwDg4O3e6rO3nOnj3LzZs3GTp0qFG7nZ0dTk5OFBUVAeDu7o6tra1hdxU9/fr8Rx55pFNxtysvLycrK4szZ860O9ampiaAO86uCyFMR4pxIYS4BxQUFKAoiuGlQbi1rvxuy1v6I2dnZ1QqFTU1NXeM6amTS7uTR//B5/Lly0btWq2W6upqwxaHZmZmTJkyhcLCQnQ6nWFdd35+PiqVyrBOvaNxt0tLS2POnDk4OTm1O1b9s3Rxceny/QoheocsUxFCiAFIp9Px66+/0tLSwsmTJ1myZAlubm5G64a9vLyorq4mNzeX5uZmqqqqWs26Ajg5OVFWVkZJSQlarZbm5mby8/NNtrWhRqPBw8ODixcvtvn14uJiXFxciIyMbPW1qKgoXFxcOHbs2F376W4ed3d3Jk6cyJYtWygsLKS+vp7S0lIWLlwIwLx58wyxSUlJVFRUkJyczPXr1/nmm29IT08nOjoab2/vTsfBrfXfW7duZenSpXe9V/2z9PPzu2usEKJvSTEuhBB9bMOGDYwdOxaAhIQEwsPD2bhxI2+//TZwaznC+fPn2bJlC/Hx8QA8++yznD171pCjoaEBPz8/rKysCAoKYtSoURw4cMBonfXixYuZOHEiL7zwAt7e3qxatcqwTOH2FwxjYmJwdnbG19eXKVOmUF1d3SfPoT1Tp06lqKiozR1E2tsru6mpicrKSqMDcu6ku3lUKhU5OTlERUUxb948HB0d8fX15cKFC+zatYugoCBD7MMPP8xnn33G559/zv3338/MmTOZO3cumzZtMsrZ0TiAN954g7CwsFb7krfl+++/Z9iwYW0udRFCmJZKkRMAhOg1KpWK7OzsVvs9i4ErIiICgJycHJONYdGiReTk5HD16lWTjaEzdu7cSWRkZKcOnCkuLsbHx4dt27YxZ86cDl+n0+mYMGEC0dHRzJ07tyvD7dE8/cHVq1cZPnw4q1evNny466j+8PMuxD0uR2bGhRBiAGrv5cZ7gZeXFytXrmTlypXU1dV16JqbN2+Sm5uLVqslKiqqy333VJ7+IiUlhTFjxhAbG2vqoQgh2iDFuBD9xJtvvml4ce3dd9/t8/5PnDhBVFQU7u7uWFpa8sADD/Doo4+yevXqLudsbGwkLi6OIUOGoNFo2L9/fw+OWNzrEhMTiYiIICoqqt2XOfUKCgrYtWsX+fn5dz25sy/y9AcZGRkcP36cffv2dXkvdiFE75JiXIh+4q9//WurI9L7yqlTpwgMDGTIkCEcOHCAmpoaDh8+zLPPPktBQUGX87711lvs37+f06dPk5mZ2eEZTnFny5YtY9u2bdTU1ODu7s4nn3xi6iH1qjVr1hAbG8vrr79+19iQkBC2b99u2K+7q3oqj6nl5eXR2NhIQUEBjo6Oph6OEOIOZGtDIQRvvvkmDg4OZGZmGtpGjRrFqlWrmDlzZpfz5ubm4u/vj4ODAwsWLACgvr6ekJAQk33wGOjWrl3L2rVrTT2MPhUaGkpoaKiphzHghIeHEx4ebuphCCHuQmbGhRBcvXqVmpqaVrtoWFhYdGsf5osXL7b60/j7779PZWVll3MKIYQQ9xIpxoXoxzIzM7G2tmbQoEE88cQTuLi4YG5ujrW1NY8//jhBQUGMGDECtVqNg4MDr732mtH1+/fv79Be0WPHjuX69etMmjSJf//73+3GKopCRkYGPj4+WFpa4ujoyLRp0zh9+rQh5osvvsDLy4vLly/z4YcfolKpsLGxYcmSJcTHx3Pu3DlUKhVeXl7dvseDBw/i6+uLvb09arUaPz8/PvvsMwA++OADbGxsUKlUODo6kpuby9GjRxk5ciT33Xcff/rTnzrz7RBCCCF6nBTjQvRjS5Ys4dVXX0VRFDZt2sTPP/9MeXk5Tz31FD/88AOJiYn88MMPVFdX8+KLL5Kens6JEycM1+t33NDpdO3289prr+Hv78+JEycYP348Dz/8MG+88Uab+02npKSQmJjI8uXLqayspLCwkNLSUoKCgqioqADg6aefNhyo8uKLL6IoCnV1dWRmZvLcc8/h6emJoigUFxd3+x4rKiqIjIykpKSEsrIybGxsmD17NgDR0dF8//33aDQawsPDmTZtGv7+/syePZvNmzeTlZXV7e+REEII0R1SjAsxQPj6+qLRaLj//vt54YUXAHBzc+OBBx5Ao9EY9mK+fYZ66tSp1NbWkpSU1G5uKysrDh8+zN///nd+97vf8dNPP5GQkICPjw//+te/DHH19fVkZGQwY8YM5syZg729PX5+frz77rtcuXKFzZs39/k9zpo1i+TkZBwdHXFyciIsLIyrV69SVVUFgI+PD2+//TYffvgh27dv5+OPP6axsdHodEQhhBDCVOQFTiEGIAsLCwBaWloMbfq12c3NzV3KaW5uTmxsLLGxsRw5coTU1FRyc3OJiIjgzJkzODo6UlRURF1dHf7+/kbXjh07FgsLC44cOdLFO2qtq/eoj7l9H+4FCxbwxRdfsGjRIv7whz90eweSb7/91nAYirg7/VHs8swGnm+//ZZx48aZehhC3NNkZlwI0crvf/97/vnPfxITE0NVVRUHDhwA4Nq1awDY2Ni0usbBwQGtVtun4wTYu3cvEyZMYPDgwVhaWrZaU663Zs0a6urq5OVRIYQQ/YrMjAshmDlzJtnZ2ZiZGf+X8Oc//5lNmzZx48YN4FbBDbRZdF+7do3hw4f3/mBvc+HCBaZPn86MGTPYunUrQ4cOZf369a0K8ubmZuLi4sjIyCA+Pp7Vq1eTnJzc5X7HjRsnx4N3ws6dO4mMjJRnNgDJXzOE6H1SjAshaGxs5KeffuKRRx4xaj9z5gyAoX306NHY2Nhw9OhRo7gjR47Q1NTEE0880TcD/n+nTp2iubmZxYsX4+HhAYBKpWoV98orrzB//nxmzJjBpUuXWLVqFaGhoQQEBPTpeIUQQojfkmUqQtzD8vPzO7S1IcD06dPZuXMn165do6amhry8PP72t78RHh5uKMbVajXx8fHs3r2bjz76iNraWk6dOkVMTAyurq4sXLjwrv04OTlRVlZGSUkJWq22y2vc4dbLnQBffvklDQ0NnD17ttW69XfeeYdhw4YxY8YM4NahOb6+vsyePZva2tou9y2EEEL0CEUI0WsAJTs7u0Oxb731luLi4qIAirW1tTJjxgwlMzNT0Wg0CqA8+OCDysGDB5XU1FTF3t5eARQXFxdl+/btyo4dOwzXOjo6Kh9//LGiKIqyb98+xdbWVlm9enW7fX/++edKZGSk4unpqVhaWioWFhaKt7e3kpKSojQ0NBjF6nQ6JT09XXnooYcUc3NzxdHRUZk+fbpy5swZQ0xJSYny2GOPKYBiZmamPP7448onn3yiKIqiHDt2TBk5cqRiZWWljB8/XklMTOzWPSYkJChOTk6Kg4ODEhERoWzYsEEBFE9PT2XMmDGKSqVSnJyclMOHDyuKoihLly5VBg0apACKvb29cvTo0Y59M//frFmzlFmzZnXqmv912dnZivy6GZjk512IXrdTpSiKYpqPAULc+1QqFdnZ2Tz//POmHoroIfo1tLL+ueP0a8bl183AIz/vQvS6HFmmIoQQQgghhIlIMS6EEKJf+/LLL0lMTESn0zF9+nTc3NxQq9UMGzaM8PBwTp482al8PZVHLysri7Fjx2Jra8vIkSN56aWXKC8vbxV36NAhnnzySTQaDa6uriQkJNDY2NiluAkTJqBSqdr8p996dM+ePaSlpRntuS+E6H+kGBdCCNFvJScns27dOpYtW4ZOp+PgwYNkZWVRXV3NoUOHqK+v56mnnqKsrKzDOXsqD0B2djazZ88mIiKCixcvkpeXR2FhIZMnTzY6sKqoqIjQ0FBCQkKoqqpi9+7dbN26lZiYGKN8HY1rz/jx4wEICwtDrVYTEhJiOCNACNH/SDEuhBADTH19PYGBgQO+j7tJTU1lx44d7Ny5E1tbWwACAgIYP348Go0Gd3d31qxZQ01NDR988EGncvdUnn/84x8MHTqUV199FXt7e8aMGcNf/vIXjh8/brSzz6pVqxgyZAgrVqzA2tqagIAAEhIS+OCDDzh9+nSn49RqNbW1tSiKYvRv4cKFRvvsx8XF8eijjzJlyhSjDwdCiP5DinEhhBhg3n///V4/SbQv+mhPcXExSUlJrFixArVaDYCZmRmffvqpUZx+f/lz5851OHdP5QEoLS3F1dXVaH/7ESNGAPDLL78A0NLSwt69ewkODjaKmzx5MoqikJeX16k4gP379xs+oNw+lh9//JFJkyYZtaekpHD8+HEyMzM7dW9CiL4hxbgQQvQyRVHIyMjAx8cHS0tLHB0dmTZtmtFMZ2xsLBYWFgwZMsTQ9vLLL2NtbY1KpeLKlSsALFmyhPj4eM6dO4dKpcLLy4t169ahVqtxdnZm0aJFuLq6olarCQwMNJqd7U4fcKsA7Oi+9d21bt06FEUhLCys3bj6+noA7OzsutVfV/N4eHi0+tCiXy+uL/DPnz9PXV2dYV98PU9PTwDDWvWOxt1JamoqcXFxrdodHR0JDg4mMzNTdrQRoh+SYlwIIXpZSkoKiYmJLF++nMrKSgoLCyktLSUoKIiKigrgVvH52y0w33nnHVasWGHUlpmZyXPPPYenpyeKolBcXExsbCzR0dHcuHGDuLg4SkpKOHbsGC0tLTz99NOUlpZ2uw/A8CKgTqfruYdzB3v37sXb2xuNRtNu3HfffQf8d510V3U1z7JlyygvL2f9+vVotVqKiorIzMzkmWeeYdy4ccB/i/PfzmSr1WqsrKwMPwMdjWvLpUuXKCgoYObMmW1+/bHHHuPSpUucOHGiU/cnhOh9UowLIUQvqq+vJyMjgxkzZjBnzhzs7e3x8/Pj3Xff5cqVK2zevLnH+jIzMzPMvvv6+rJx40a0Wi3btm3rkfxTp06ltraWpKSkHsl3J9evX+fnn382zAi3paKigh07dhAXF0dAQMBdZ9B7K09wcDAJCQnExsZiZ2fH6NGj0Wq1vPfee4YY/U4o9913X6vrzc3NDbPyHY1rS2pqKq+88gqDBrX9a/2hhx4C4NSpUx28MyFEX5FiXAghelFRURF1dXX4+/sbtY8dOxYLCwujZSQ9zd/fH41GY7QcZiCorKxEUZR2Z8UDAgKIi4tj2rRp5OfnY25u3qW+uptn+fLlbN68ma+++oq6ujrOnz9PYGAgAQEBhr9I6Ne8t/UCZVNTE1ZWVp2K+62ysjL27NlDdHT0Hcepf5btza4LIUxDinEhhOhF+i3l9Hs/387BwQGtVtur/VtaWlJVVdWrffS0hoYG4NbY78TZ2Zmvv/6a9evXY29v3+W+upPn8uXLpKWlsWDBAiZNmoS1tTXu7u5s2bKFsrIy0tPTAQxr9Gtra42uv3HjBg0NDbi6unYq7rfS0tKYP3++oZhvi76Q1z9bIUT/YWbqAQghxL3MwcEBoM2i+9q1awwfPrzX+m5ubu71PnqDvnBs77CawYMHG55td3Qnz9mzZ7l58yZDhw41arezs8PJyYmioiIA3N3dsbW1Neyuoqdfi//II490Ku525eXlZGVlcebMmXbH2tTUBHDH2XUhhOlIMS6EEL1o9OjR2NjYcPToUaP2I0eO0NTUxBNPPGFoMzMzo7m5ucf6LigoQFEUw4uEvdFHb3B2dkalUlFTU3PHmN9uTdhV3cmj/5Bz+fJlo3atVkt1dbVhi0MzMzOmTJlCYWEhOp3OsK47Pz8flUplWKfe0bjbpaWlMWfOHJycnNodq/5Zuri4dPl+hRC9Q5apCCFEL1Kr1cTHx7N7924++ugjamtrOXXqFDExMbi6urJw4UJDrJeXF9XV1eTm5tLc3ExVVVWrWVIAJycnysrKKCkpQavVGoprnU7Hr7/+SktLCydPnmTJkiW4ubkZrSXuTh/5+fl9srWhRqPBw8ODixcvtvn14uJiXFxciIyMbPW1qKgoXFxcOHbs2F376W4ed3d3Jk6cyJYtWygsLKS+vp7S0lLD93TevHmG2KSkJCoqKkhOTub69et88803pKenEx0djbe3d6fj4Nb6761bt7J06dK73qv+Wfr5+d01VgjRt6QYF0KIXpacnMzatWtZuXIlDzzwAMHBwTz44IMUFBRgbW1tiFu8eDETJ07khRdewNvbm1WrVhmWFdz+QmBMTAzOzs74+voyZcoUqqurgVvrgf38/LCysiIoKIhRo0Zx4MABo7XX3e2jr0ydOpWioqI2dxBpb6/spqYmKisrjQ7IuZPu5lGpVOTk5BAVFcW8efNwdHTE19eXCxcusGvXLoKCggyxDz/8MJ999hmff/45999/PzNnzmTu3Lls2rTJKGdH4wDeeOMNwsLCWu1L3pbvv/+eYcOGtbnURQhhWipFTgAQoteoVCqys7Nb7e0sBq6IiAgAcnJyTDwSY4sWLSInJ4erV6+aeiit7Ny5k8jIyE4dOFNcXIyPjw/btm1jzpw5Hb5Op9MxYcIEoqOjmTt3bleG26N5+oOrV68yfPhwVq9eTXx8fKeu7a8/70LcQ3JkZlwIIe4R7b3wONB4eXmxcuVKVq5cSV1dXYeuuXnzJrm5uWi1WqKiorrcd0/l6S9SUlIYM2YMsbGxph6KEKINUowLIYTolxITE4mIiCAqKqrdlzn1CgoK2LVrF/n5+Xc9ubMv8vQHGRkZHD9+nH379nV5L3YhRO+SYlwIIQa4ZcuWsW3bNmpqanB3d+eTTz4x9ZB6zJo1a4iNjeX111+/a2xISAjbt2837NfdVT2Vx9Ty8vJobGykoKAAR0dHUw9HCHEHsrWhEEIMcGvXrmXt2rWmHkavCQ0NJTQ01NTDGHDCw8MJDw839TCEEHchM+NCCCGEEEKYiBTjQgghhBBCmIgU40IIIYQQQpiIFONCCCGEEEKYiBTjQgghhBBCmIicwClEL1KpVKYeghBCdMusWbPkBE4hek+ObG0oRC/Kzs429RCEEKJbRowYYeohCHFPk5lxIYQQQgghTCNH1owLIYQQQghhIlKMCyGEEEIIYSJSjAshhBBCCGEiZoC8Ii2EEEIIIUTf+/b/AByvrIq3yQSdAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<IPython.core.display.Image object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 185
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0WfW_jOjdxO7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 기존 학습 weight 로드\n",
        "model.load_weights(\"songys_chat_transformer.hdf5\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6ztQryROd6sq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def do_predict(model, n_seq, string):\n",
        "    \"\"\"\n",
        "    입력에 대한 답변 생성하는 함수\n",
        "    :param model: model\n",
        "    :param n_seq: 디코더 입력 개수\n",
        "    :param string: 입력 문자열\n",
        "    \"\"\"\n",
        "    # enc_token 생성: <string tokens>, [PAD] tokens\n",
        "    enc_token = vocab.encode_as_ids(string)\n",
        "    # dec_token 생성: [BOS], [PAD] tokens\n",
        "    dec_token = [vocab.bos_id()]\n",
        "    dec_token += [0] * (n_seq - len(dec_token))\n",
        "    dec_token = dec_token[:n_seq]\n",
        "\n",
        "    response = []\n",
        "    for i in range(n_seq - 1):\n",
        "        # model 실행\n",
        "        output = model.predict((np.array([enc_token]), np.array([dec_token])))\n",
        "        # decoder의 마지막 위치의 token 예측 값\n",
        "        word_id = int(np.argmax(output, axis=2)[0][i])\n",
        "        # [EOS] 토큰이 생성되면 종료\n",
        "        if word_id == vocab.eos_id():\n",
        "            break\n",
        "        # 예측된 token을 응답에 저장\n",
        "        response.append(word_id)\n",
        "        # 예측된 token을 decoder의 다음 입력으로 저장\n",
        "        dec_token[i + 1] = word_id\n",
        "    \n",
        "    # 생성된 token을 문자열로 변경\n",
        "    return vocab.decode_ids(response)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CAGU6rovd2eB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "d925e1ee-47b8-467e-e454-8d53ceed428b"
      },
      "source": [
        "while True:\n",
        "    print(\"input > \", end=\"\")\n",
        "    string = str(input())\n",
        "    if len(string) == 0:\n",
        "        break\n",
        "    print(f\"output > {do_predict(model, 20, string)}\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "input > 안녕 만나서 반가워\n",
            "output > 반갑습니다.\n",
            "input > 너는 누구니?\n",
            "output > 저는 마음을 이어주는 위로봇입니다.\n",
            "input > 오늘 날씨 어때>\n",
            "output > 따뜻해졌죠.\n",
            "input > \n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}