text/transformer_model_for_language_understanding.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "transformer_model_for_language_understanding.ipynb",
      "provenance": [],
      "machine_shape": "hm",
      "authorship_tag": "ABX9TyNCnp31Ce3rlAqbCDobdXLJ",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "7cc1b732a9024a9b882498f92e521138": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2afe448f79284e0999b718962b741581",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_80c34ae2ec6b486592b3fcd83372e16d",
              "IPY_MODEL_e76773a0929b4eaaae08fa1552278456"
            ]
          }
        },
        "2afe448f79284e0999b718962b741581": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "80c34ae2ec6b486592b3fcd83372e16d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_caa9c01271bb43e09028fbef25c0d7bb",
            "_dom_classes": [],
            "description": "Dl Completed...: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e45b9a19f16e4edf8dd739fd4d93121f"
          }
        },
        "e76773a0929b4eaaae08fa1552278456": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6519540071814ccca749fcb22caf4b14",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:08&lt;00:00,  8.21s/ url]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b686bbee0deb4f3aa7710fd43c74037e"
          }
        },
        "caa9c01271bb43e09028fbef25c0d7bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e45b9a19f16e4edf8dd739fd4d93121f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6519540071814ccca749fcb22caf4b14": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b686bbee0deb4f3aa7710fd43c74037e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "61dcab3f2cc54d45aa4ee4d014e7413a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b6a04b0815f74104b02c4f0e2a9224bd",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_83a39dcdb0f2439f8dcb056f8a1096e0",
              "IPY_MODEL_9c6aece9f5cb4105a5097fce0f5abd0c"
            ]
          }
        },
        "b6a04b0815f74104b02c4f0e2a9224bd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "83a39dcdb0f2439f8dcb056f8a1096e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1b4ebb814eab47ae9cb36eecccd4ecb4",
            "_dom_classes": [],
            "description": "Dl Size...: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_14e212528a3448c593aa5de77f612f8b"
          }
        },
        "9c6aece9f5cb4105a5097fce0f5abd0c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_77c1051a66af41c4a30e5888d3029f69",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 124/124 [00:08&lt;00:00, 15.15 MiB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_30a289a9717a419a8ef3a8003e7429ad"
          }
        },
        "1b4ebb814eab47ae9cb36eecccd4ecb4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "14e212528a3448c593aa5de77f612f8b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "77c1051a66af41c4a30e5888d3029f69": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "30a289a9717a419a8ef3a8003e7429ad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "55e25f689d8c4278ade497cc2e951875": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e29e84cbc72244b7a21075169d6aed54",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0b2c48bc7f8149bfac9f6808078f5fe2",
              "IPY_MODEL_8a96188a9f294748ba57921bf0320071"
            ]
          }
        },
        "e29e84cbc72244b7a21075169d6aed54": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0b2c48bc7f8149bfac9f6808078f5fe2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_312dd420a48e44149b36c750434d3ce8",
            "_dom_classes": [],
            "description": "Extraction completed...: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b7d94e79c88a4fdf8f827c98aec4c5fa"
          }
        },
        "8a96188a9f294748ba57921bf0320071": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f5d74f6b576342c5ad9fe7e81aace6be",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:08&lt;00:00,  8.15s/ file]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_880ae36f9e1c48aabd37b51a56025c8f"
          }
        },
        "312dd420a48e44149b36c750434d3ce8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b7d94e79c88a4fdf8f827c98aec4c5fa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f5d74f6b576342c5ad9fe7e81aace6be": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "880ae36f9e1c48aabd37b51a56025c8f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8545fb7eb8234f3a863c4d9ea0ae55f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_42e89a9ea4434d32816af55486f42b5d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6e943baf2f4e49518eb7aa5014487004",
              "IPY_MODEL_bdd478d161224f3aaf37f7a95d4d23d6"
            ]
          }
        },
        "42e89a9ea4434d32816af55486f42b5d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6e943baf2f4e49518eb7aa5014487004": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f06ae8c837354ac4994d56f7619a483b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_74b84738b51247fdaefe01c9fb4b78a3"
          }
        },
        "bdd478d161224f3aaf37f7a95d4d23d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_77240b4de076496986a327eaf31138f0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 51785/0 [00:13&lt;00:00, 3863.80 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9095aa92552a4051b41e687da93ef461"
          }
        },
        "f06ae8c837354ac4994d56f7619a483b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "74b84738b51247fdaefe01c9fb4b78a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "77240b4de076496986a327eaf31138f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9095aa92552a4051b41e687da93ef461": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4867e4134cd3459483610b1724931a79": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_54b02c5db3c84d1ea58ed1a28123286a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_54f5239cafdd42a39d0462e74448d72e",
              "IPY_MODEL_4a903ae5e48a4641adb5e72344e9ca8f"
            ]
          }
        },
        "54b02c5db3c84d1ea58ed1a28123286a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "54f5239cafdd42a39d0462e74448d72e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_be90260d3ce44cb4a6ff85a8dd591019",
            "_dom_classes": [],
            "description": " 31%",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 51785,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 16169,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d9a2ff3fb00b4b7ca9d8225000137d62"
          }
        },
        "4a903ae5e48a4641adb5e72344e9ca8f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1f48e6d2467545f8bcc3e0be9cc5e7a5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 16169/51785 [00:20&lt;00:00, 161684.76 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_37e3378e0e0c4d42a0000b29bef4d012"
          }
        },
        "be90260d3ce44cb4a6ff85a8dd591019": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d9a2ff3fb00b4b7ca9d8225000137d62": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1f48e6d2467545f8bcc3e0be9cc5e7a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "37e3378e0e0c4d42a0000b29bef4d012": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "db079f35d1cc424386f2bc6ce157959e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_cf19113a43c64f9f966f2f32293fd7a1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_114a2395f3564b4cbe6f94ac5be4a24b",
              "IPY_MODEL_f2ad2275fc8e433494f7174089bd859a"
            ]
          }
        },
        "cf19113a43c64f9f966f2f32293fd7a1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "114a2395f3564b4cbe6f94ac5be4a24b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7ede5a61ea334b92aad4ac6b7d7e033b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_25641deddf304a81bcc43eed10b70341"
          }
        },
        "f2ad2275fc8e433494f7174089bd859a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4816fc969e4242e3ad1c288e24fae2d6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1193/0 [00:00&lt;00:00, 3110.82 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_96d44c972f4a4156a8851b6385bd7507"
          }
        },
        "7ede5a61ea334b92aad4ac6b7d7e033b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "25641deddf304a81bcc43eed10b70341": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4816fc969e4242e3ad1c288e24fae2d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "96d44c972f4a4156a8851b6385bd7507": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "83f3341e42d745c19b03b7ad080eef19": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_fd8c7c31214541e99ffad9683d00decf",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6f8715dc12934c6d9072ec5c585159ba",
              "IPY_MODEL_40335f6734394369b09997c88a787cf5"
            ]
          }
        },
        "fd8c7c31214541e99ffad9683d00decf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6f8715dc12934c6d9072ec5c585159ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d4d12c9cc2674689abcbf186176aa8d7",
            "_dom_classes": [],
            "description": "  0%",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 1193,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 0,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d166e60cbc2f4364b28e634bb1f05d06"
          }
        },
        "40335f6734394369b09997c88a787cf5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e4cf3660a0f4404481a73f7c3bd866fb",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 0/1193 [00:08&lt;?, ? examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c4cf887b21404b42a1e4d4adb71ea37e"
          }
        },
        "d4d12c9cc2674689abcbf186176aa8d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d166e60cbc2f4364b28e634bb1f05d06": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e4cf3660a0f4404481a73f7c3bd866fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c4cf887b21404b42a1e4d4adb71ea37e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b9bd6a5c8cd04278bfdb8327c2a89d57": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_938f921762be4e0b83b5925c5197c829",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_094a367c482447fd97a9847fc249be62",
              "IPY_MODEL_41a084c4755347a38c0d6ca957af9508"
            ]
          }
        },
        "938f921762be4e0b83b5925c5197c829": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "094a367c482447fd97a9847fc249be62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_effe876f1d8a44b99de6647be056aad3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_18cf3825490440f79f7b69a583f2a313"
          }
        },
        "41a084c4755347a38c0d6ca957af9508": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4625194cbd774ba5aa76e14fa1e4fe3e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1803/0 [00:00&lt;00:00, 3251.82 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_21e61d88068a4cc1bacf088dcf2c94da"
          }
        },
        "effe876f1d8a44b99de6647be056aad3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "18cf3825490440f79f7b69a583f2a313": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4625194cbd774ba5aa76e14fa1e4fe3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "21e61d88068a4cc1bacf088dcf2c94da": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "95849b7cafb34c3a8ad5c322188d5c63": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_92146479b1b547b59877fead3f67f68d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_772a3bc781014e06bf29f0fccdff09ff",
              "IPY_MODEL_a1f67c719d184d2a82c762a7202c5cca"
            ]
          }
        },
        "92146479b1b547b59877fead3f67f68d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "772a3bc781014e06bf29f0fccdff09ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2ed6595939d540c4979f0dbcd0bb5596",
            "_dom_classes": [],
            "description": "  0%",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 1803,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 0,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f30a04d5c09f4a1db501e640e77fedf2"
          }
        },
        "a1f67c719d184d2a82c762a7202c5cca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_578ec8868ede44dfac928c3c91bd62dd",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 0/1803 [00:07&lt;?, ? examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c57dc0f1704e49b493d5e40d413543cc"
          }
        },
        "2ed6595939d540c4979f0dbcd0bb5596": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f30a04d5c09f4a1db501e640e77fedf2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "578ec8868ede44dfac928c3c91bd62dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c57dc0f1704e49b493d5e40d413543cc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/dwdb/tensorflow2.0-tutorial/blob/master/text/transformer_model_for_language_understanding.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rw_8PPMJISLc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "0e830c05-a8fd-44ba-c434-8b6c24cc0b84"
      },
      "source": [
        "import time\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "import tensorflow_datasets as tfds\n",
        "\n",
        "tf.__version__"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'2.2.0'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0czQcdjJIn0G",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "BUFFER_SIZE = 20000\n",
        "BATCH_SIZE = 64\n",
        "MAX_LENGTH = 40\n",
        "MAX_PE_LENGTH = 200\n",
        "NUM_LAYERS = 4\n",
        "D_MODEL = 128\n",
        "DFF = 512\n",
        "NUM_HEADS = 8\n",
        "DROPOUT_RATE = 0.1\n",
        "EPOCHS = 20\n",
        "\n",
        "train_step_signature = [\n",
        "    tf.TensorSpec(shape=(None, None), dtype=tf.int64),\n",
        "    tf.TensorSpec(shape=(None, None), dtype=tf.int64),\n",
        "]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B7nCgk4kIt1z",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 409,
          "referenced_widgets": [
            "7cc1b732a9024a9b882498f92e521138",
            "2afe448f79284e0999b718962b741581",
            "80c34ae2ec6b486592b3fcd83372e16d",
            "e76773a0929b4eaaae08fa1552278456",
            "caa9c01271bb43e09028fbef25c0d7bb",
            "e45b9a19f16e4edf8dd739fd4d93121f",
            "6519540071814ccca749fcb22caf4b14",
            "b686bbee0deb4f3aa7710fd43c74037e",
            "61dcab3f2cc54d45aa4ee4d014e7413a",
            "b6a04b0815f74104b02c4f0e2a9224bd",
            "83a39dcdb0f2439f8dcb056f8a1096e0",
            "9c6aece9f5cb4105a5097fce0f5abd0c",
            "1b4ebb814eab47ae9cb36eecccd4ecb4",
            "14e212528a3448c593aa5de77f612f8b",
            "77c1051a66af41c4a30e5888d3029f69",
            "30a289a9717a419a8ef3a8003e7429ad",
            "55e25f689d8c4278ade497cc2e951875",
            "e29e84cbc72244b7a21075169d6aed54",
            "0b2c48bc7f8149bfac9f6808078f5fe2",
            "8a96188a9f294748ba57921bf0320071",
            "312dd420a48e44149b36c750434d3ce8",
            "b7d94e79c88a4fdf8f827c98aec4c5fa",
            "f5d74f6b576342c5ad9fe7e81aace6be",
            "880ae36f9e1c48aabd37b51a56025c8f",
            "8545fb7eb8234f3a863c4d9ea0ae55f0",
            "42e89a9ea4434d32816af55486f42b5d",
            "6e943baf2f4e49518eb7aa5014487004",
            "bdd478d161224f3aaf37f7a95d4d23d6",
            "f06ae8c837354ac4994d56f7619a483b",
            "74b84738b51247fdaefe01c9fb4b78a3",
            "77240b4de076496986a327eaf31138f0",
            "9095aa92552a4051b41e687da93ef461",
            "4867e4134cd3459483610b1724931a79",
            "54b02c5db3c84d1ea58ed1a28123286a",
            "54f5239cafdd42a39d0462e74448d72e",
            "4a903ae5e48a4641adb5e72344e9ca8f",
            "be90260d3ce44cb4a6ff85a8dd591019",
            "d9a2ff3fb00b4b7ca9d8225000137d62",
            "1f48e6d2467545f8bcc3e0be9cc5e7a5",
            "37e3378e0e0c4d42a0000b29bef4d012",
            "db079f35d1cc424386f2bc6ce157959e",
            "cf19113a43c64f9f966f2f32293fd7a1",
            "114a2395f3564b4cbe6f94ac5be4a24b",
            "f2ad2275fc8e433494f7174089bd859a",
            "7ede5a61ea334b92aad4ac6b7d7e033b",
            "25641deddf304a81bcc43eed10b70341",
            "4816fc969e4242e3ad1c288e24fae2d6",
            "96d44c972f4a4156a8851b6385bd7507",
            "83f3341e42d745c19b03b7ad080eef19",
            "fd8c7c31214541e99ffad9683d00decf",
            "6f8715dc12934c6d9072ec5c585159ba",
            "40335f6734394369b09997c88a787cf5",
            "d4d12c9cc2674689abcbf186176aa8d7",
            "d166e60cbc2f4364b28e634bb1f05d06",
            "e4cf3660a0f4404481a73f7c3bd866fb",
            "c4cf887b21404b42a1e4d4adb71ea37e",
            "b9bd6a5c8cd04278bfdb8327c2a89d57",
            "938f921762be4e0b83b5925c5197c829",
            "094a367c482447fd97a9847fc249be62",
            "41a084c4755347a38c0d6ca957af9508",
            "effe876f1d8a44b99de6647be056aad3",
            "18cf3825490440f79f7b69a583f2a313",
            "4625194cbd774ba5aa76e14fa1e4fe3e",
            "21e61d88068a4cc1bacf088dcf2c94da",
            "95849b7cafb34c3a8ad5c322188d5c63",
            "92146479b1b547b59877fead3f67f68d",
            "772a3bc781014e06bf29f0fccdff09ff",
            "a1f67c719d184d2a82c762a7202c5cca",
            "2ed6595939d540c4979f0dbcd0bb5596",
            "f30a04d5c09f4a1db501e640e77fedf2",
            "578ec8868ede44dfac928c3c91bd62dd",
            "c57dc0f1704e49b493d5e40d413543cc"
          ]
        },
        "outputId": "bd8aeb43-3069-4139-8072-f0a20359f442"
      },
      "source": [
        "dataset_name = 'ted_hrlr_translate/pt_to_en'\n",
        "examples, meta_data = tfds.load(dataset_name, with_info=True, as_supervised=True)\n",
        "train_examples, valid_examples = examples['train'], examples['validation']\n",
        "\n",
        "tokenizer_pt = tfds.features.text.SubwordTextEncoder.build_from_corpus(\n",
        "    (pt.numpy() for pt, en in train_examples), 2 ** 13)\n",
        "\n",
        "tokenizer_en = tfds.features.text.SubwordTextEncoder.build_from_corpus(\n",
        "    (en.numpy() for pt, en in train_examples), 2 ** 13)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[1mDownloading and preparing dataset ted_hrlr_translate/pt_to_en/1.0.0 (download: 124.94 MiB, generated: Unknown size, total: 124.94 MiB) to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0...\u001b[0m\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7cc1b732a9024a9b882498f92e521138",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Dl Completed...', max=1.0, style=Progre…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "61dcab3f2cc54d45aa4ee4d014e7413a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Dl Size...', max=1.0, style=ProgressSty…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "55e25f689d8c4278ade497cc2e951875",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Extraction completed...', max=1.0, styl…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8545fb7eb8234f3a863c4d9ea0ae55f0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\rShuffling and writing examples to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0.incomplete8WWBDU/ted_hrlr_translate-train.tfrecord\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4867e4134cd3459483610b1724931a79",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=51785.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "db079f35d1cc424386f2bc6ce157959e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Shuffling and writing examples to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0.incomplete8WWBDU/ted_hrlr_translate-validation.tfrecord\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "83f3341e42d745c19b03b7ad080eef19",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1193.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b9bd6a5c8cd04278bfdb8327c2a89d57",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Shuffling and writing examples to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0.incomplete8WWBDU/ted_hrlr_translate-test.tfrecord\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "95849b7cafb34c3a8ad5c322188d5c63",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1803.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[1mDataset ted_hrlr_translate downloaded and prepared to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0. Subsequent calls will reuse this data.\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iAfTYy8AIx6e",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def tf_encode(lang1, lang2):\n",
        "    def encode(lang1, lang2):\n",
        "        encoded1 = [tokenizer_pt.vocab_size] + tokenizer_pt.encode(\n",
        "            lang1.numpy()) + [tokenizer_pt.vocab_size + 1]\n",
        "        encoded2 = [tokenizer_en.vocab_size] + tokenizer_en.encode(\n",
        "            lang2.numpy()) + [tokenizer_en.vocab_size + 1]\n",
        "        return encoded1, encoded2\n",
        "\n",
        "    lang1, lang2 = tf.py_function(encode, [lang1, lang2], [tf.int64, tf.int64])\n",
        "    lang1.set_shape([None])\n",
        "    lang2.set_shape([None])\n",
        "    return lang1, lang2\n",
        "\n",
        "\n",
        "def filter_max_length(x, y):\n",
        "    return tf.logical_and(tf.size(x) <= MAX_LENGTH, tf.size(y) <= MAX_LENGTH)\n",
        "\n",
        "\n",
        "# 保留长度较短的句子、添加起止标记、缓冲加速读取\n",
        "# ！！！先map再filter，否则filter不起作用\n",
        "train_dataset = train_examples \\\n",
        "    .map(tf_encode) \\\n",
        "    .filter(filter_max_length) \\\n",
        "    .cache() \\\n",
        "    .shuffle(BUFFER_SIZE) \\\n",
        "    .padded_batch(BATCH_SIZE, padded_shapes=([None], [None])) \\\n",
        "    .prefetch(tf.data.experimental.AUTOTUNE)\n",
        "\n",
        "valid_dataset = valid_examples \\\n",
        "    .map(tf_encode) \\\n",
        "    .filter(filter_max_length) \\\n",
        "    .padded_batch(BATCH_SIZE, padded_shapes=([None], [None]))\n",
        "\n",
        "INP_VOCAB_SIZE = tokenizer_pt.vocab_size + 2\n",
        "TAR_VOCAB_SIZE = tokenizer_en.vocab_size + 2"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZhGtXMFgJGb6",
        "colab_type": "text"
      },
      "source": [
        "# Selt Attention"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WlKqooUbJExI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def scaled_dot_product_attention(q, k, v, mask):\n",
        "    \"\"\"\n",
        "    The transformer takes three inputs: Q (query), K (key), V (value).\n",
        "\n",
        "    The equation used to calculate the attention weights is:\n",
        "        Attention(Q, K, V) = softmax(Q·K^T/sqrt(d_k))V\n",
        "\n",
        "    sqrt(d_k): This is done because for large valus of depth, the dot product grows large\n",
        "    in magnitude pushing the softmax function where it has small gradients resulting in a\n",
        "    very hard softmax.\n",
        "\n",
        "    The mask is multiplied with 1e-9 (close to negative infinity), and large negative\n",
        "    inputs to softmax are near zero in the output.\n",
        "\n",
        "    :param q: shape=(..., seq_len, depth)\n",
        "    :param k: shape=(..., seq_len, depth)\n",
        "    :param v: shape=(..., seq_len, depth_v)\n",
        "    :param mask: shape=(..., seq_len, seq_len)\n",
        "        multi_head => (batch_size, 1, 1, seq_len), 利用广播性质mask\n",
        "    \"\"\"\n",
        "    # (..., seq_len, seq_len)\n",
        "    matmul_qk = tf.matmul(q, k, transpose_b=True)\n",
        "    dk = tf.cast(tf.shape(k)[-1], tf.float32)\n",
        "    scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n",
        "\n",
        "    if mask is not None:\n",
        "        # padding_mask = (batch_size, 1, 1, seq_len)\n",
        "        # combined_mask = (batch_size, 1, seq_len, seq_len)\n",
        "        scaled_attention_logits += (mask * -1e9)\n",
        "\n",
        "    # softmax is normalized on the last axis (seq_len) so that the scores add up to 1.\n",
        "    # (..., seq_len, seq_len), 对最后二维矩阵的行做归一化\n",
        "    attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)\n",
        "    # (..., seq_len, depth_v)\n",
        "    output = tf.matmul(attention_weights, v)\n",
        "\n",
        "    return output, attention_weights\n",
        "\n",
        "\n",
        "class MultiHeadAttention(tf.keras.layers.Layer):\n",
        "    def __init__(self, d_model, num_heads):\n",
        "        super(MultiHeadAttention, self).__init__()\n",
        "        # number of scaled dot-product attentions\n",
        "        self.num_heads = num_heads\n",
        "        # dimension of output\n",
        "        self.d_model = d_model\n",
        "        assert d_model % num_heads == 0\n",
        "        self.depth = d_model // num_heads\n",
        "        self.wq = tf.keras.layers.Dense(d_model)\n",
        "        self.wk = tf.keras.layers.Dense(d_model)\n",
        "        self.wv = tf.keras.layers.Dense(d_model)\n",
        "        self.dense = tf.keras.layers.Dense(d_model)\n",
        "\n",
        "    def split_heads(self, x, batch_size):\n",
        "        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n",
        "        # 转置变换使得后期计算可以充分利用“广播”性质，加速计算\n",
        "        return tf.transpose(x, perm=[0, 2, 1, 3])\n",
        "\n",
        "    def call(self, v, k=None, q=None, mask=None):\n",
        "        batch_size = tf.shape(q)[0]\n",
        "        # shape=(batch_size, num_heads, seq_len, depth)\n",
        "        q = self.split_heads(self.wq(q), batch_size)\n",
        "        k = self.split_heads(self.wk(k), batch_size)\n",
        "        v = self.split_heads(self.wv(v), batch_size)\n",
        "\n",
        "        # scaled_attention.shape=(batch_size, num_heads, seq_len_q, depth)\n",
        "        # attention_weights.shape=(batch_size, num_heads, seq_len_q, seq_len_k)\n",
        "        scaled_attention, attention_weights = scaled_dot_product_attention(q, k, v, mask)\n",
        "        # (batch_size, seq_len_q, num_heads, depth)\n",
        "        scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])\n",
        "        # (batch_size, seq_len_q, d_model), d_model=num_heads*depth\n",
        "        concat_attention = tf.reshape(scaled_attention, (batch_size, -1, self.d_model))\n",
        "\n",
        "        # W_O matrix, (batch_size, seq_len_q, d_model)\n",
        "        output = self.dense(concat_attention)\n",
        "        return output, attention_weights"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QgEXfcaQJOZP",
        "colab_type": "text"
      },
      "source": [
        "# Point wise feed forward network"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HhzpnAizJJzo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def point_wise_feed_forward_network(d_model, dff):\n",
        "    \"\"\"\n",
        "    Point wise feed forward network, consist of two fully-connected layers with a RELU\n",
        "    activation in between.\n",
        "    \"\"\"\n",
        "    ff_mdoel = tf.keras.Sequential([\n",
        "        tf.keras.layers.Dense(dff, activation='relu'),\n",
        "        tf.keras.layers.Dense(d_model)])\n",
        "    return ff_mdoel"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zojVlDxfJNI_",
        "colab_type": "text"
      },
      "source": [
        "# Positional Encoding"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5A1w9KyRJYtn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def positional_encoding(position, d_model):\n",
        "    \"\"\"\n",
        "    Positional encoding is added to give the model some information about the relative\n",
        "    position of the words in the sentence.\n",
        "\n",
        "    The formula for calculating the positional encoding is as follwing:\n",
        "        PE(pos,2i)   = sin(pos/10000^(2i/d_model))\n",
        "        PE(pos,2i+1) = cos(pos/10000^(2i/d_model))\n",
        "    \"\"\"\n",
        "    positions = np.arange(position)[:, np.newaxis]\n",
        "    evens = np.arange(d_model)[np.newaxis, :] // 2\n",
        "    angle_rates = 1 / np.power(10000, 2 * evens / np.float32(d_model))\n",
        "    # shape=(seq_len, n_dimension)\n",
        "    angle_rads = positions * angle_rates\n",
        "    # even positions\n",
        "    angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n",
        "    # odd position\n",
        "    angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n",
        "    # shape=(1, position, n_dimension)\n",
        "    encoding = angle_rads[np.newaxis, ...]\n",
        "    return tf.cast(encoding, dtype=tf.float32)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nfE3DJ6_JipL",
        "colab_type": "text"
      },
      "source": [
        "# Mask"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mOZZrzMBJlnQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def create_padding_mask(sequence):\n",
        "    \"\"\"\n",
        "    Mask all the pad tokens in the batch of sequence.\n",
        "    It ensures that the model does not treat padding as the input.\n",
        "    \"\"\"\n",
        "    # shape=(batch_size, seq_len)\n",
        "    mask = tf.cast(tf.math.equal(sequence, 0), tf.float32)\n",
        "    # shape=(batch_size, 1, 1, seq_len)\n",
        "    mask = mask[:, tf.newaxis, tf.newaxis, :]\n",
        "    return mask\n",
        "\n",
        "\n",
        "def create_look_ahead_mask(size):\n",
        "    \"\"\"\n",
        "    Mask the future tokens in a sequence. Used for decoding that mean the current output\n",
        "    of decoder is independent of the future output.\n",
        "    \"\"\"\n",
        "    # upper triangular matrix, shape=(seq_len, seq_len)\n",
        "    mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n",
        "    return mask\n",
        "\n",
        "\n",
        "def create_combined_mask(sequence):\n",
        "    \"\"\"\n",
        "    创建联合掩码\n",
        "    主要用于解码器输入，需同时考虑padding_mask和look_ahead_mask\n",
        "    \"\"\"\n",
        "    look_ahead_mask = create_look_ahead_mask(tf.shape(sequence)[1])\n",
        "    padding_mask = create_padding_mask(sequence)\n",
        "    combined_mask = tf.maximum(padding_mask, look_ahead_mask)\n",
        "    return combined_mask"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UGhlubfBJnwl",
        "colab_type": "text"
      },
      "source": [
        "# Encoder"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xtxMiX7hJtHl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class EncoderLayer(tf.keras.layers.Layer):\n",
        "    \"\"\"\n",
        "    Each encoder layer consists of sublayers.\n",
        "        1. Multi-head attention (with padding mask)\n",
        "        2. Point wise feed forward networks.\n",
        "\n",
        "    Each of these sublayers has a residual connection around it followed by a layer\n",
        "    normalization. Residual connections help in avoiding the vanishing gradient problem\n",
        "    in deep networks.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, d_model, num_heads, dff, rate=0.1):\n",
        "        super(EncoderLayer, self).__init__()\n",
        "\n",
        "        self.mha = MultiHeadAttention(d_model, num_heads)\n",
        "        self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
        "\n",
        "        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "\n",
        "        self.dropout1 = tf.keras.layers.Dropout(rate)\n",
        "        self.dropout2 = tf.keras.layers.Dropout(rate)\n",
        "\n",
        "    def call(self, x, training, mask):\n",
        "        # all the shape=(batch_size, seq_len)\n",
        "        attention_output, _ = self.mha(x, x, x, mask)\n",
        "        attention_output = self.dropout1(attention_output, training=training)\n",
        "        output1 = self.layernorm1(x + attention_output)\n",
        "\n",
        "        ffn_output = self.ffn(output1)\n",
        "        ffn_output = self.dropout2(ffn_output, training=training)\n",
        "        output2 = self.layernorm2(output1 + ffn_output)\n",
        "        return output2\n",
        "\n",
        "\n",
        "class Encoder(tf.keras.layers.Layer):\n",
        "\n",
        "    def __init__(self, num_layers, d_model, num_heads, dff, vocab_size, maximum_position_encoding,\n",
        "                 rate=0.1):\n",
        "        super(Encoder, self).__init__()\n",
        "\n",
        "        self.d_model = d_model\n",
        "        self.num_layers = num_layers\n",
        "        self.embedding = tf.keras.layers.Embedding(vocab_size, d_model)\n",
        "        self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)\n",
        "        self.enc_layers = [\n",
        "            EncoderLayer(d_model, num_heads, dff, rate) for _ in range(num_layers)]\n",
        "        self.dropout = tf.keras.layers.Dropout(rate)\n",
        "\n",
        "    def call(self, x, training, mask):\n",
        "        x = self.embedding(x)\n",
        "        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
        "        x += self.pos_encoding[:, :tf.shape(x)[1], :]\n",
        "        x = self.dropout(x, training=training)\n",
        "        for enc_layer in self.enc_layers:\n",
        "            x = enc_layer(x, training, mask)\n",
        "        return x"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FZ-fep95JtsC",
        "colab_type": "text"
      },
      "source": [
        "# Decoder"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B2PH2isgJyCQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class DecoderLayer(tf.keras.layers.Layer):\n",
        "    def __init__(self, d_model, num_heads, dff, rate=0.1):\n",
        "        super(DecoderLayer, self).__init__()\n",
        "\n",
        "        self.mha1 = MultiHeadAttention(d_model, num_heads)\n",
        "        self.mha2 = MultiHeadAttention(d_model, num_heads)\n",
        "        self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
        "\n",
        "        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "        self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "\n",
        "        self.dropout1 = tf.keras.layers.Dropout(rate)\n",
        "        self.dropout2 = tf.keras.layers.Dropout(rate)\n",
        "        self.dropout3 = tf.keras.layers.Dropout(rate)\n",
        "\n",
        "    def call(self, x, enc_output, training, combined_mask, enc_mask):\n",
        "        attn1, block1 = self.mha1(x, x, x, combined_mask)\n",
        "        attn1 = self.dropout1(attn1, training=training)\n",
        "        out1 = self.layernorm1(attn1 + x)\n",
        "\n",
        "        attn2, block2 = self.mha2(enc_output, enc_output, out1, enc_mask)\n",
        "        attn2 = self.dropout2(attn2, training=training)\n",
        "        out2 = self.layernorm2(attn2 + out1)\n",
        "\n",
        "        ffn_output = self.ffn(out2)\n",
        "        ffn_output = self.dropout3(ffn_output, training=training)\n",
        "        out3 = self.layernorm3(ffn_output + out2)\n",
        "        return out3, block1, block2\n",
        "\n",
        "\n",
        "class Decoder(tf.keras.layers.Layer):\n",
        "    def __init__(self, num_layers, d_model, num_heads, dff, vocab_size, maximum_position_encoding,\n",
        "                 rate=0.1):\n",
        "        super(Decoder, self).__init__()\n",
        "\n",
        "        self.d_model = d_model\n",
        "        self.num_layers = num_layers\n",
        "        self.embedding = tf.keras.layers.Embedding(vocab_size, d_model)\n",
        "        self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)\n",
        "        self.dec_layers = [\n",
        "            DecoderLayer(d_model, num_heads, dff, rate) for _ in range(num_layers)]\n",
        "        self.dropout = tf.keras.layers.Dropout(rate)\n",
        "\n",
        "    def call(self, x, enc_output, training, combined_mask, enc_mask):\n",
        "        x = self.embedding(x)\n",
        "        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
        "        x += self.pos_encoding[:, :tf.shape(x)[1], :]\n",
        "        x = self.dropout(x, training=training)\n",
        "\n",
        "        attention_weights = {}\n",
        "        for i, dec_layer in enumerate(self.dec_layers):\n",
        "            x, block1, block2 = dec_layer(x, enc_output, training, combined_mask, enc_mask)\n",
        "            attention_weights['decoder_layer{}_block1'.format(i + 1)] = block1\n",
        "            attention_weights['decoder_layer{}_block2'.format(i + 1)] = block2\n",
        "\n",
        "        return x, attention_weights"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FyjcnobcJzrM",
        "colab_type": "text"
      },
      "source": [
        "# Transformer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZeWdEMxeJ5F7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Transformer(tf.keras.Model):\n",
        "    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, target_vocab_size,\n",
        "                 pe_input, pe_target, rate=0.1):\n",
        "        super(Transformer, self).__init__()\n",
        "        self.encoder = Encoder(num_layers, d_model, num_heads, dff, input_vocab_size,\n",
        "                               pe_input, rate)\n",
        "        self.decoder = Decoder(num_layers, d_model, num_heads, dff, target_vocab_size,\n",
        "                               pe_target, rate)\n",
        "        self.final_layer = tf.keras.layers.Dense(target_vocab_size)\n",
        "\n",
        "    def call(self, inp, tar, training):\n",
        "        padding_mask = create_padding_mask(inp)\n",
        "        combined_mask = create_combined_mask(tar)\n",
        "\n",
        "        # (batch_size, inp_seq_len, d_model)\n",
        "        enc_output = self.encoder(inp, training, padding_mask)\n",
        "        # dec_output.shape == (batch_size, tar_seq_len, d_model)\n",
        "        dec_output, attention_weights = self.decoder(\n",
        "            tar, enc_output, training, combined_mask, padding_mask)\n",
        "\n",
        "        # (batch_size, tar_seq_len, target_vocab_size)\n",
        "        final_output = self.final_layer(dec_output)\n",
        "        return final_output, attention_weights"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "koi4BkA4J5lV",
        "colab_type": "text"
      },
      "source": [
        "# Custom Learning Rate"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RpiMOpszJ9Gm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
        "    def __init__(self, d_model, warmup_steps=4000):\n",
        "        super(CustomSchedule, self).__init__()\n",
        "        self.d_model = tf.cast(d_model, tf.float32)\n",
        "        self.warmup_steps = warmup_steps\n",
        "\n",
        "    def __call__(self, step):\n",
        "        arg1 = tf.math.rsqrt(step)\n",
        "        arg2 = step * (self.warmup_steps ** -1.5)\n",
        "        return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)\n",
        "\n",
        "    def get_config(self):\n",
        "        pass"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RCAtQzoNJ9bQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Translator:\n",
        "\n",
        "    def __init__(self):\n",
        "        self.transformer = Transformer(NUM_LAYERS, D_MODEL, NUM_HEADS, DFF, INP_VOCAB_SIZE,\n",
        "                                       TAR_VOCAB_SIZE, MAX_PE_LENGTH, MAX_PE_LENGTH,\n",
        "                                       DROPOUT_RATE)\n",
        "\n",
        "        self.train_loss = tf.keras.metrics.Mean('train_loss')\n",
        "        self.train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy('train_accuracy')\n",
        "        self.loss_object = tf.keras.losses.SparseCategoricalCrossentropy(True, reduction='none')\n",
        "        self.learning_rate = CustomSchedule(D_MODEL)\n",
        "        self.optimizer = tf.keras.optimizers.Adam(self.learning_rate, 0.9, 0.98, 1e-9)\n",
        "\n",
        "        ckpt = tf.train.Checkpoint(transformer=self.transformer, optimizer=self.optimizer)\n",
        "        self.ckpt_manager = tf.train.CheckpointManager(ckpt, \"./checkpoints/train\", 5)\n",
        "        if self.ckpt_manager.latest_checkpoint:\n",
        "            ckpt.restore(self.ckpt_manager.latest_checkpoint)\n",
        "            print('Latest checkpoint restored!!')\n",
        "\n",
        "    def loss_function(self, real, pred):\n",
        "        loss = self.loss_object(real, pred)\n",
        "        mask = tf.cast(tf.math.logical_not(tf.math.equal(real, 0)), dtype=loss.dtype)\n",
        "        loss = tf.reduce_sum(loss * mask) / tf.reduce_sum(mask)\n",
        "        return loss\n",
        "\n",
        "    @tf.function(input_signature=train_step_signature)\n",
        "    def train_step(self, inp, tar):\n",
        "        tar_inp, tar_real = tar[:, :-1], tar[:, 1:]\n",
        "        with tf.GradientTape() as tape:\n",
        "            predictions, _ = self.transformer(inp, tar_inp, True)\n",
        "            loss = self.loss_function(tar_real, predictions)\n",
        "\n",
        "        gradients = tape.gradient(loss, self.transformer.trainable_variables)\n",
        "        self.optimizer.apply_gradients(\n",
        "            zip(gradients, self.transformer.trainable_variables))\n",
        "        self.train_loss(loss)\n",
        "        self.train_accuracy(tar_real, predictions)\n",
        "\n",
        "    def train(self):\n",
        "        for epoch in range(EPOCHS):\n",
        "            start = time.time()\n",
        "            self.train_loss.reset_states()\n",
        "            self.train_accuracy.reset_states()\n",
        "\n",
        "            for (batch, (inp, tar)) in enumerate(train_dataset):\n",
        "                self.train_step(inp, tar)\n",
        "\n",
        "                if batch % 50 == 0:\n",
        "                    print('Epoch {} Batch {} Loss {:.4f} Accuracy {:.4f}'.format(\n",
        "                        epoch + 1, batch, self.train_loss.result(), self.train_accuracy.result()))\n",
        "\n",
        "            if (epoch + 1) % 5 == 0:\n",
        "                ckpt_save_path = self.ckpt_manager.save()\n",
        "                print('Saving checkpoint for epoch {} at {}'.format(epoch + 1, ckpt_save_path))\n",
        "\n",
        "            print('Epoch {} Loss {:.4f} Accuracy {:.4f}'.format(\n",
        "                epoch + 1, self.train_loss.result(), self.train_accuracy.result()))\n",
        "            print('Time taken for 1 epoch: {} secs\\n'.format(time.time() - start))\n",
        "\n",
        "    def evaluate(self, inp_sentence):\n",
        "        start_token = [tokenizer_pt.vocab_size]\n",
        "        end_token = [tokenizer_pt.vocab_size + 1]\n",
        "\n",
        "        # inp sentence is portuguese, hence adding the start and end token\n",
        "        inp_sentence = start_token + tokenizer_pt.encode(inp_sentence) + end_token\n",
        "        encoder_input = tf.expand_dims(inp_sentence, 0)\n",
        "\n",
        "        # as the target is english, the first word to the transformer should be the\n",
        "        # english start token.\n",
        "        decoder_input = [tokenizer_en.vocab_size]\n",
        "        output = tf.expand_dims(decoder_input, 0)\n",
        "\n",
        "        for i in range(MAX_LENGTH):\n",
        "            predictions, attention_weights = self.transformer(encoder_input, output, False)\n",
        "            # select the last word from the seq_len dimension\n",
        "            predictions = predictions[:, -1:, :]  # (batch_size, 1, vocab_size)\n",
        "            predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)\n",
        "            # return the result if the predicted_id is equal to the end token\n",
        "            if predicted_id == tokenizer_en.vocab_size + 1:\n",
        "                return tf.squeeze(output, axis=0), attention_weights\n",
        "            # concatentate the predicted_id to the output which is given to the decoder\n",
        "            # as its input.\n",
        "\n",
        "            output = tf.concat([output, predicted_id], axis=-1)\n",
        "\n",
        "        return tf.squeeze(output, axis=0), attention_weights\n",
        "\n",
        "    def translate(self, sentence, plot=False):\n",
        "        result, attention_weights = self.evaluate(sentence)\n",
        "        predicted_sentence = tokenizer_en.decode([i for i in result if i < tokenizer_en.vocab_size])\n",
        "        print('Input: {}'.format(sentence))\n",
        "        print('Predicted translation: {}'.format(predicted_sentence))\n",
        "        if plot:\n",
        "            self.plot_attention_weights(attention_weights, sentence, result, plot)\n",
        "\n",
        "    @staticmethod\n",
        "    def plot_attention_weights(attention, sentence, result, layer):\n",
        "        fig = plt.figure(figsize=(16, 8))\n",
        "        sentence = tokenizer_pt.encode(sentence)\n",
        "        attention = tf.squeeze(attention[layer], axis=0)\n",
        "        for head in range(attention.shape[0]):\n",
        "            ax = fig.add_subplot(2, 4, head + 1)\n",
        "            # plot the attention weights\n",
        "            ax.matshow(attention[head][:-1, :], cmap='viridis')\n",
        "            fontdict = {'fontsize': 10}\n",
        "            ax.set_xticks(range(len(sentence) + 2))\n",
        "            ax.set_yticks(range(len(result)))\n",
        "            ax.set_ylim(len(result) - 1.5, -0.5)\n",
        "            ax.set_xticklabels(\n",
        "                ['<start>'] + [tokenizer_pt.decode([i]) for i in sentence] + ['<end>'],\n",
        "                fontdict=fontdict, rotation=90)\n",
        "            ax.set_yticklabels(\n",
        "                [tokenizer_en.decode([i]) for i in result if i < tokenizer_en.vocab_size],\n",
        "                fontdict=fontdict)\n",
        "            ax.set_xlabel('Head {}'.format(head + 1))\n",
        "        plt.tight_layout()\n",
        "        plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NALuRSbcKDsV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "translator = Translator()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0tlPr4xcKJO6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "d8bfdff2-821b-4ae4-d38c-8456c9a824ba"
      },
      "source": [
        "translator.train()"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1 Batch 0 Loss 8.9957 Accuracy 0.0000\n",
            "Epoch 1 Batch 50 Loss 8.9446 Accuracy 0.0040\n",
            "Epoch 1 Batch 100 Loss 8.8452 Accuracy 0.0142\n",
            "Epoch 1 Batch 150 Loss 8.7388 Accuracy 0.0172\n",
            "Epoch 1 Batch 200 Loss 8.6105 Accuracy 0.0188\n",
            "Epoch 1 Batch 250 Loss 8.4553 Accuracy 0.0197\n",
            "Epoch 1 Batch 300 Loss 8.2794 Accuracy 0.0235\n",
            "Epoch 1 Batch 350 Loss 8.0924 Accuracy 0.0272\n",
            "Epoch 1 Batch 400 Loss 7.9115 Accuracy 0.0300\n",
            "Epoch 1 Batch 450 Loss 7.7468 Accuracy 0.0326\n",
            "Epoch 1 Batch 500 Loss 7.6046 Accuracy 0.0357\n",
            "Epoch 1 Batch 550 Loss 7.4719 Accuracy 0.0394\n",
            "Epoch 1 Batch 600 Loss 7.3464 Accuracy 0.0432\n",
            "Epoch 1 Batch 650 Loss 7.2288 Accuracy 0.0469\n",
            "Epoch 1 Batch 700 Loss 7.1135 Accuracy 0.0505\n",
            "Epoch 1 Loss 7.1091 Accuracy 0.0506\n",
            "Time taken for 1 epoch: 96.10432004928589 secs\n",
            "\n",
            "Epoch 2 Batch 0 Loss 5.4537 Accuracy 0.0963\n",
            "Epoch 2 Batch 50 Loss 5.4867 Accuracy 0.1041\n",
            "Epoch 2 Batch 100 Loss 5.4189 Accuracy 0.1063\n",
            "Epoch 2 Batch 150 Loss 5.3708 Accuracy 0.1080\n",
            "Epoch 2 Batch 200 Loss 5.3319 Accuracy 0.1093\n",
            "Epoch 2 Batch 250 Loss 5.2874 Accuracy 0.1111\n",
            "Epoch 2 Batch 300 Loss 5.2485 Accuracy 0.1129\n",
            "Epoch 2 Batch 350 Loss 5.2109 Accuracy 0.1148\n",
            "Epoch 2 Batch 400 Loss 5.1753 Accuracy 0.1165\n",
            "Epoch 2 Batch 450 Loss 5.1412 Accuracy 0.1181\n",
            "Epoch 2 Batch 500 Loss 5.1102 Accuracy 0.1196\n",
            "Epoch 2 Batch 550 Loss 5.0817 Accuracy 0.1211\n",
            "Epoch 2 Batch 600 Loss 5.0544 Accuracy 0.1225\n",
            "Epoch 2 Batch 650 Loss 5.0292 Accuracy 0.1237\n",
            "Epoch 2 Batch 700 Loss 5.0062 Accuracy 0.1248\n",
            "Epoch 2 Loss 5.0050 Accuracy 0.1248\n",
            "Time taken for 1 epoch: 55.58212232589722 secs\n",
            "\n",
            "Epoch 3 Batch 0 Loss 4.8276 Accuracy 0.1389\n",
            "Epoch 3 Batch 50 Loss 4.5943 Accuracy 0.1428\n",
            "Epoch 3 Batch 100 Loss 4.5921 Accuracy 0.1441\n",
            "Epoch 3 Batch 150 Loss 4.5857 Accuracy 0.1437\n",
            "Epoch 3 Batch 200 Loss 4.5721 Accuracy 0.1444\n",
            "Epoch 3 Batch 250 Loss 4.5624 Accuracy 0.1448\n",
            "Epoch 3 Batch 300 Loss 4.5499 Accuracy 0.1457\n",
            "Epoch 3 Batch 350 Loss 4.5396 Accuracy 0.1464\n",
            "Epoch 3 Batch 400 Loss 4.5284 Accuracy 0.1473\n",
            "Epoch 3 Batch 450 Loss 4.5165 Accuracy 0.1478\n",
            "Epoch 3 Batch 500 Loss 4.5037 Accuracy 0.1484\n",
            "Epoch 3 Batch 550 Loss 4.4921 Accuracy 0.1488\n",
            "Epoch 3 Batch 600 Loss 4.4774 Accuracy 0.1496\n",
            "Epoch 3 Batch 650 Loss 4.4633 Accuracy 0.1505\n",
            "Epoch 3 Batch 700 Loss 4.4493 Accuracy 0.1512\n",
            "Epoch 3 Loss 4.4486 Accuracy 0.1513\n",
            "Time taken for 1 epoch: 56.605058908462524 secs\n",
            "\n",
            "Epoch 4 Batch 0 Loss 4.1625 Accuracy 0.1661\n",
            "Epoch 4 Batch 50 Loss 4.1605 Accuracy 0.1659\n",
            "Epoch 4 Batch 100 Loss 4.1445 Accuracy 0.1662\n",
            "Epoch 4 Batch 150 Loss 4.1349 Accuracy 0.1656\n",
            "Epoch 4 Batch 200 Loss 4.1294 Accuracy 0.1662\n",
            "Epoch 4 Batch 250 Loss 4.1133 Accuracy 0.1673\n",
            "Epoch 4 Batch 300 Loss 4.1000 Accuracy 0.1680\n",
            "Epoch 4 Batch 350 Loss 4.0875 Accuracy 0.1693\n",
            "Epoch 4 Batch 400 Loss 4.0718 Accuracy 0.1704\n",
            "Epoch 4 Batch 450 Loss 4.0559 Accuracy 0.1712\n",
            "Epoch 4 Batch 500 Loss 4.0398 Accuracy 0.1722\n",
            "Epoch 4 Batch 550 Loss 4.0238 Accuracy 0.1733\n",
            "Epoch 4 Batch 600 Loss 4.0066 Accuracy 0.1746\n",
            "Epoch 4 Batch 650 Loss 3.9911 Accuracy 0.1756\n",
            "Epoch 4 Batch 700 Loss 3.9760 Accuracy 0.1767\n",
            "Epoch 4 Loss 3.9758 Accuracy 0.1767\n",
            "Time taken for 1 epoch: 55.74897360801697 secs\n",
            "\n",
            "Epoch 5 Batch 0 Loss 3.6489 Accuracy 0.1989\n",
            "Epoch 5 Batch 50 Loss 3.6185 Accuracy 0.1935\n",
            "Epoch 5 Batch 100 Loss 3.6151 Accuracy 0.1940\n",
            "Epoch 5 Batch 150 Loss 3.6108 Accuracy 0.1953\n",
            "Epoch 5 Batch 200 Loss 3.6017 Accuracy 0.1965\n",
            "Epoch 5 Batch 250 Loss 3.5896 Accuracy 0.1974\n",
            "Epoch 5 Batch 300 Loss 3.5804 Accuracy 0.1982\n",
            "Epoch 5 Batch 350 Loss 3.5693 Accuracy 0.1992\n",
            "Epoch 5 Batch 400 Loss 3.5561 Accuracy 0.1999\n",
            "Epoch 5 Batch 450 Loss 3.5498 Accuracy 0.2003\n",
            "Epoch 5 Batch 500 Loss 3.5421 Accuracy 0.2012\n",
            "Epoch 5 Batch 550 Loss 3.5308 Accuracy 0.2021\n",
            "Epoch 5 Batch 600 Loss 3.5197 Accuracy 0.2030\n",
            "Epoch 5 Batch 650 Loss 3.5120 Accuracy 0.2037\n",
            "Epoch 5 Batch 700 Loss 3.5031 Accuracy 0.2043\n",
            "Saving checkpoint for epoch 5 at ./checkpoints/train/ckpt-1\n",
            "Epoch 5 Loss 3.5034 Accuracy 0.2044\n",
            "Time taken for 1 epoch: 56.28933310508728 secs\n",
            "\n",
            "Epoch 6 Batch 0 Loss 3.0906 Accuracy 0.2242\n",
            "Epoch 6 Batch 50 Loss 3.1609 Accuracy 0.2194\n",
            "Epoch 6 Batch 100 Loss 3.1562 Accuracy 0.2201\n",
            "Epoch 6 Batch 150 Loss 3.1704 Accuracy 0.2192\n",
            "Epoch 6 Batch 200 Loss 3.1747 Accuracy 0.2191\n",
            "Epoch 6 Batch 250 Loss 3.1674 Accuracy 0.2202\n",
            "Epoch 6 Batch 300 Loss 3.1623 Accuracy 0.2208\n",
            "Epoch 6 Batch 350 Loss 3.1565 Accuracy 0.2214\n",
            "Epoch 6 Batch 400 Loss 3.1473 Accuracy 0.2226\n",
            "Epoch 6 Batch 450 Loss 3.1410 Accuracy 0.2230\n",
            "Epoch 6 Batch 500 Loss 3.1395 Accuracy 0.2231\n",
            "Epoch 6 Batch 550 Loss 3.1335 Accuracy 0.2235\n",
            "Epoch 6 Batch 600 Loss 3.1260 Accuracy 0.2240\n",
            "Epoch 6 Batch 650 Loss 3.1207 Accuracy 0.2246\n",
            "Epoch 6 Batch 700 Loss 3.1142 Accuracy 0.2251\n",
            "Epoch 6 Loss 3.1135 Accuracy 0.2251\n",
            "Time taken for 1 epoch: 55.68229794502258 secs\n",
            "\n",
            "Epoch 7 Batch 0 Loss 2.8520 Accuracy 0.2360\n",
            "Epoch 7 Batch 50 Loss 2.7777 Accuracy 0.2449\n",
            "Epoch 7 Batch 100 Loss 2.7730 Accuracy 0.2451\n",
            "Epoch 7 Batch 150 Loss 2.7631 Accuracy 0.2430\n",
            "Epoch 7 Batch 200 Loss 2.7622 Accuracy 0.2431\n",
            "Epoch 7 Batch 250 Loss 2.7645 Accuracy 0.2433\n",
            "Epoch 7 Batch 300 Loss 2.7689 Accuracy 0.2435\n",
            "Epoch 7 Batch 350 Loss 2.7590 Accuracy 0.2436\n",
            "Epoch 7 Batch 400 Loss 2.7524 Accuracy 0.2438\n",
            "Epoch 7 Batch 450 Loss 2.7488 Accuracy 0.2442\n",
            "Epoch 7 Batch 500 Loss 2.7413 Accuracy 0.2445\n",
            "Epoch 7 Batch 550 Loss 2.7330 Accuracy 0.2451\n",
            "Epoch 7 Batch 600 Loss 2.7273 Accuracy 0.2454\n",
            "Epoch 7 Batch 650 Loss 2.7207 Accuracy 0.2458\n",
            "Epoch 7 Batch 700 Loss 2.7154 Accuracy 0.2464\n",
            "Epoch 7 Loss 2.7151 Accuracy 0.2465\n",
            "Time taken for 1 epoch: 55.66211199760437 secs\n",
            "\n",
            "Epoch 8 Batch 0 Loss 2.2993 Accuracy 0.2790\n",
            "Epoch 8 Batch 50 Loss 2.4181 Accuracy 0.2653\n",
            "Epoch 8 Batch 100 Loss 2.4049 Accuracy 0.2643\n",
            "Epoch 8 Batch 150 Loss 2.4010 Accuracy 0.2640\n",
            "Epoch 8 Batch 200 Loss 2.4114 Accuracy 0.2635\n",
            "Epoch 8 Batch 250 Loss 2.4102 Accuracy 0.2641\n",
            "Epoch 8 Batch 300 Loss 2.4008 Accuracy 0.2642\n",
            "Epoch 8 Batch 350 Loss 2.4033 Accuracy 0.2643\n",
            "Epoch 8 Batch 400 Loss 2.4055 Accuracy 0.2642\n",
            "Epoch 8 Batch 450 Loss 2.4027 Accuracy 0.2642\n",
            "Epoch 8 Batch 500 Loss 2.4006 Accuracy 0.2641\n",
            "Epoch 8 Batch 550 Loss 2.4013 Accuracy 0.2642\n",
            "Epoch 8 Batch 600 Loss 2.3969 Accuracy 0.2646\n",
            "Epoch 8 Batch 650 Loss 2.3939 Accuracy 0.2649\n",
            "Epoch 8 Batch 700 Loss 2.3916 Accuracy 0.2652\n",
            "Epoch 8 Loss 2.3920 Accuracy 0.2652\n",
            "Time taken for 1 epoch: 56.11734127998352 secs\n",
            "\n",
            "Epoch 9 Batch 0 Loss 2.1990 Accuracy 0.3214\n",
            "Epoch 9 Batch 50 Loss 2.1432 Accuracy 0.2778\n",
            "Epoch 9 Batch 100 Loss 2.1333 Accuracy 0.2781\n",
            "Epoch 9 Batch 150 Loss 2.1419 Accuracy 0.2788\n",
            "Epoch 9 Batch 200 Loss 2.1470 Accuracy 0.2788\n",
            "Epoch 9 Batch 250 Loss 2.1448 Accuracy 0.2801\n",
            "Epoch 9 Batch 300 Loss 2.1452 Accuracy 0.2797\n",
            "Epoch 9 Batch 350 Loss 2.1452 Accuracy 0.2795\n",
            "Epoch 9 Batch 400 Loss 2.1446 Accuracy 0.2794\n",
            "Epoch 9 Batch 450 Loss 2.1453 Accuracy 0.2798\n",
            "Epoch 9 Batch 500 Loss 2.1458 Accuracy 0.2798\n",
            "Epoch 9 Batch 550 Loss 2.1483 Accuracy 0.2798\n",
            "Epoch 9 Batch 600 Loss 2.1486 Accuracy 0.2794\n",
            "Epoch 9 Batch 650 Loss 2.1533 Accuracy 0.2789\n",
            "Epoch 9 Batch 700 Loss 2.1565 Accuracy 0.2788\n",
            "Epoch 9 Loss 2.1565 Accuracy 0.2787\n",
            "Time taken for 1 epoch: 56.20690417289734 secs\n",
            "\n",
            "Epoch 10 Batch 0 Loss 1.9004 Accuracy 0.3125\n",
            "Epoch 10 Batch 50 Loss 1.9452 Accuracy 0.2894\n",
            "Epoch 10 Batch 100 Loss 1.9374 Accuracy 0.2910\n",
            "Epoch 10 Batch 150 Loss 1.9453 Accuracy 0.2910\n",
            "Epoch 10 Batch 200 Loss 1.9516 Accuracy 0.2894\n",
            "Epoch 10 Batch 250 Loss 1.9543 Accuracy 0.2898\n",
            "Epoch 10 Batch 300 Loss 1.9623 Accuracy 0.2901\n",
            "Epoch 10 Batch 350 Loss 1.9638 Accuracy 0.2898\n",
            "Epoch 10 Batch 400 Loss 1.9637 Accuracy 0.2896\n",
            "Epoch 10 Batch 450 Loss 1.9687 Accuracy 0.2904\n",
            "Epoch 10 Batch 500 Loss 1.9683 Accuracy 0.2903\n",
            "Epoch 10 Batch 550 Loss 1.9704 Accuracy 0.2903\n",
            "Epoch 10 Batch 600 Loss 1.9745 Accuracy 0.2904\n",
            "Epoch 10 Batch 650 Loss 1.9774 Accuracy 0.2904\n",
            "Epoch 10 Batch 700 Loss 1.9775 Accuracy 0.2905\n",
            "Saving checkpoint for epoch 10 at ./checkpoints/train/ckpt-2\n",
            "Epoch 10 Loss 1.9772 Accuracy 0.2906\n",
            "Time taken for 1 epoch: 55.99298071861267 secs\n",
            "\n",
            "Epoch 11 Batch 0 Loss 1.7059 Accuracy 0.3041\n",
            "Epoch 11 Batch 50 Loss 1.7791 Accuracy 0.3040\n",
            "Epoch 11 Batch 100 Loss 1.7904 Accuracy 0.3020\n",
            "Epoch 11 Batch 150 Loss 1.7914 Accuracy 0.3013\n",
            "Epoch 11 Batch 200 Loss 1.8020 Accuracy 0.3004\n",
            "Epoch 11 Batch 250 Loss 1.8059 Accuracy 0.2993\n",
            "Epoch 11 Batch 300 Loss 1.8083 Accuracy 0.2990\n",
            "Epoch 11 Batch 350 Loss 1.8090 Accuracy 0.2996\n",
            "Epoch 11 Batch 400 Loss 1.8127 Accuracy 0.2992\n",
            "Epoch 11 Batch 450 Loss 1.8167 Accuracy 0.2992\n",
            "Epoch 11 Batch 500 Loss 1.8186 Accuracy 0.2988\n",
            "Epoch 11 Batch 550 Loss 1.8237 Accuracy 0.2985\n",
            "Epoch 11 Batch 600 Loss 1.8268 Accuracy 0.2984\n",
            "Epoch 11 Batch 650 Loss 1.8301 Accuracy 0.2984\n",
            "Epoch 11 Batch 700 Loss 1.8335 Accuracy 0.2984\n",
            "Epoch 11 Loss 1.8334 Accuracy 0.2984\n",
            "Time taken for 1 epoch: 55.628891468048096 secs\n",
            "\n",
            "Epoch 12 Batch 0 Loss 1.6832 Accuracy 0.2985\n",
            "Epoch 12 Batch 50 Loss 1.6536 Accuracy 0.3117\n",
            "Epoch 12 Batch 100 Loss 1.6521 Accuracy 0.3118\n",
            "Epoch 12 Batch 150 Loss 1.6562 Accuracy 0.3105\n",
            "Epoch 12 Batch 200 Loss 1.6688 Accuracy 0.3095\n",
            "Epoch 12 Batch 250 Loss 1.6786 Accuracy 0.3085\n",
            "Epoch 12 Batch 300 Loss 1.6874 Accuracy 0.3076\n",
            "Epoch 12 Batch 350 Loss 1.6931 Accuracy 0.3077\n",
            "Epoch 12 Batch 400 Loss 1.6937 Accuracy 0.3074\n",
            "Epoch 12 Batch 450 Loss 1.6963 Accuracy 0.3072\n",
            "Epoch 12 Batch 500 Loss 1.6976 Accuracy 0.3072\n",
            "Epoch 12 Batch 550 Loss 1.7019 Accuracy 0.3069\n",
            "Epoch 12 Batch 600 Loss 1.7063 Accuracy 0.3067\n",
            "Epoch 12 Batch 650 Loss 1.7102 Accuracy 0.3064\n",
            "Epoch 12 Batch 700 Loss 1.7143 Accuracy 0.3063\n",
            "Epoch 12 Loss 1.7148 Accuracy 0.3063\n",
            "Time taken for 1 epoch: 55.598273038864136 secs\n",
            "\n",
            "Epoch 13 Batch 0 Loss 1.5609 Accuracy 0.3255\n",
            "Epoch 13 Batch 50 Loss 1.5468 Accuracy 0.3155\n",
            "Epoch 13 Batch 100 Loss 1.5457 Accuracy 0.3162\n",
            "Epoch 13 Batch 150 Loss 1.5612 Accuracy 0.3159\n",
            "Epoch 13 Batch 200 Loss 1.5685 Accuracy 0.3154\n",
            "Epoch 13 Batch 250 Loss 1.5761 Accuracy 0.3145\n",
            "Epoch 13 Batch 300 Loss 1.5782 Accuracy 0.3140\n",
            "Epoch 13 Batch 350 Loss 1.5835 Accuracy 0.3146\n",
            "Epoch 13 Batch 400 Loss 1.5875 Accuracy 0.3144\n",
            "Epoch 13 Batch 450 Loss 1.5927 Accuracy 0.3139\n",
            "Epoch 13 Batch 500 Loss 1.5968 Accuracy 0.3137\n",
            "Epoch 13 Batch 550 Loss 1.6009 Accuracy 0.3137\n",
            "Epoch 13 Batch 600 Loss 1.6061 Accuracy 0.3134\n",
            "Epoch 13 Batch 650 Loss 1.6096 Accuracy 0.3132\n",
            "Epoch 13 Batch 700 Loss 1.6138 Accuracy 0.3129\n",
            "Epoch 13 Loss 1.6141 Accuracy 0.3129\n",
            "Time taken for 1 epoch: 55.48485493659973 secs\n",
            "\n",
            "Epoch 14 Batch 0 Loss 1.4992 Accuracy 0.3014\n",
            "Epoch 14 Batch 50 Loss 1.4547 Accuracy 0.3215\n",
            "Epoch 14 Batch 100 Loss 1.4519 Accuracy 0.3217\n",
            "Epoch 14 Batch 150 Loss 1.4696 Accuracy 0.3215\n",
            "Epoch 14 Batch 200 Loss 1.4801 Accuracy 0.3205\n",
            "Epoch 14 Batch 250 Loss 1.4797 Accuracy 0.3202\n",
            "Epoch 14 Batch 300 Loss 1.4864 Accuracy 0.3200\n",
            "Epoch 14 Batch 350 Loss 1.4894 Accuracy 0.3192\n",
            "Epoch 14 Batch 400 Loss 1.4942 Accuracy 0.3194\n",
            "Epoch 14 Batch 450 Loss 1.4999 Accuracy 0.3194\n",
            "Epoch 14 Batch 500 Loss 1.5039 Accuracy 0.3190\n",
            "Epoch 14 Batch 550 Loss 1.5101 Accuracy 0.3187\n",
            "Epoch 14 Batch 600 Loss 1.5160 Accuracy 0.3184\n",
            "Epoch 14 Batch 650 Loss 1.5235 Accuracy 0.3177\n",
            "Epoch 14 Batch 700 Loss 1.5276 Accuracy 0.3180\n",
            "Epoch 14 Loss 1.5284 Accuracy 0.3180\n",
            "Time taken for 1 epoch: 56.3967559337616 secs\n",
            "\n",
            "Epoch 15 Batch 0 Loss 1.3595 Accuracy 0.3652\n",
            "Epoch 15 Batch 50 Loss 1.3589 Accuracy 0.3287\n",
            "Epoch 15 Batch 100 Loss 1.3687 Accuracy 0.3287\n",
            "Epoch 15 Batch 150 Loss 1.3812 Accuracy 0.3278\n",
            "Epoch 15 Batch 200 Loss 1.3917 Accuracy 0.3261\n",
            "Epoch 15 Batch 250 Loss 1.4018 Accuracy 0.3252\n",
            "Epoch 15 Batch 300 Loss 1.4058 Accuracy 0.3260\n",
            "Epoch 15 Batch 350 Loss 1.4105 Accuracy 0.3260\n",
            "Epoch 15 Batch 400 Loss 1.4184 Accuracy 0.3256\n",
            "Epoch 15 Batch 450 Loss 1.4235 Accuracy 0.3259\n",
            "Epoch 15 Batch 500 Loss 1.4268 Accuracy 0.3256\n",
            "Epoch 15 Batch 550 Loss 1.4327 Accuracy 0.3249\n",
            "Epoch 15 Batch 600 Loss 1.4371 Accuracy 0.3247\n",
            "Epoch 15 Batch 650 Loss 1.4422 Accuracy 0.3245\n",
            "Epoch 15 Batch 700 Loss 1.4478 Accuracy 0.3239\n",
            "Saving checkpoint for epoch 15 at ./checkpoints/train/ckpt-3\n",
            "Epoch 15 Loss 1.4482 Accuracy 0.3239\n",
            "Time taken for 1 epoch: 55.96536993980408 secs\n",
            "\n",
            "Epoch 16 Batch 0 Loss 1.2466 Accuracy 0.3218\n",
            "Epoch 16 Batch 50 Loss 1.2903 Accuracy 0.3296\n",
            "Epoch 16 Batch 100 Loss 1.3015 Accuracy 0.3329\n",
            "Epoch 16 Batch 150 Loss 1.3179 Accuracy 0.3317\n",
            "Epoch 16 Batch 200 Loss 1.3231 Accuracy 0.3317\n",
            "Epoch 16 Batch 250 Loss 1.3283 Accuracy 0.3311\n",
            "Epoch 16 Batch 300 Loss 1.3398 Accuracy 0.3305\n",
            "Epoch 16 Batch 350 Loss 1.3466 Accuracy 0.3310\n",
            "Epoch 16 Batch 400 Loss 1.3524 Accuracy 0.3309\n",
            "Epoch 16 Batch 450 Loss 1.3586 Accuracy 0.3307\n",
            "Epoch 16 Batch 500 Loss 1.3634 Accuracy 0.3306\n",
            "Epoch 16 Batch 550 Loss 1.3692 Accuracy 0.3297\n",
            "Epoch 16 Batch 600 Loss 1.3748 Accuracy 0.3294\n",
            "Epoch 16 Batch 650 Loss 1.3803 Accuracy 0.3293\n",
            "Epoch 16 Batch 700 Loss 1.3851 Accuracy 0.3287\n",
            "Epoch 16 Loss 1.3851 Accuracy 0.3288\n",
            "Time taken for 1 epoch: 55.546616077423096 secs\n",
            "\n",
            "Epoch 17 Batch 0 Loss 1.1619 Accuracy 0.3458\n",
            "Epoch 17 Batch 50 Loss 1.2438 Accuracy 0.3346\n",
            "Epoch 17 Batch 100 Loss 1.2499 Accuracy 0.3358\n",
            "Epoch 17 Batch 150 Loss 1.2621 Accuracy 0.3337\n",
            "Epoch 17 Batch 200 Loss 1.2771 Accuracy 0.3328\n",
            "Epoch 17 Batch 250 Loss 1.2839 Accuracy 0.3334\n",
            "Epoch 17 Batch 300 Loss 1.2860 Accuracy 0.3334\n",
            "Epoch 17 Batch 350 Loss 1.2893 Accuracy 0.3338\n",
            "Epoch 17 Batch 400 Loss 1.2939 Accuracy 0.3335\n",
            "Epoch 17 Batch 450 Loss 1.2984 Accuracy 0.3340\n",
            "Epoch 17 Batch 500 Loss 1.3035 Accuracy 0.3337\n",
            "Epoch 17 Batch 550 Loss 1.3070 Accuracy 0.3337\n",
            "Epoch 17 Batch 600 Loss 1.3107 Accuracy 0.3332\n",
            "Epoch 17 Batch 650 Loss 1.3165 Accuracy 0.3329\n",
            "Epoch 17 Batch 700 Loss 1.3212 Accuracy 0.3328\n",
            "Epoch 17 Loss 1.3215 Accuracy 0.3328\n",
            "Time taken for 1 epoch: 55.540674686431885 secs\n",
            "\n",
            "Epoch 18 Batch 0 Loss 1.3057 Accuracy 0.3681\n",
            "Epoch 18 Batch 50 Loss 1.1944 Accuracy 0.3446\n",
            "Epoch 18 Batch 100 Loss 1.2012 Accuracy 0.3423\n",
            "Epoch 18 Batch 150 Loss 1.2018 Accuracy 0.3402\n",
            "Epoch 18 Batch 200 Loss 1.2104 Accuracy 0.3410\n",
            "Epoch 18 Batch 250 Loss 1.2185 Accuracy 0.3389\n",
            "Epoch 18 Batch 300 Loss 1.2295 Accuracy 0.3393\n",
            "Epoch 18 Batch 350 Loss 1.2332 Accuracy 0.3396\n",
            "Epoch 18 Batch 400 Loss 1.2384 Accuracy 0.3390\n",
            "Epoch 18 Batch 450 Loss 1.2428 Accuracy 0.3386\n",
            "Epoch 18 Batch 500 Loss 1.2493 Accuracy 0.3377\n",
            "Epoch 18 Batch 550 Loss 1.2543 Accuracy 0.3373\n",
            "Epoch 18 Batch 600 Loss 1.2616 Accuracy 0.3369\n",
            "Epoch 18 Batch 650 Loss 1.2668 Accuracy 0.3367\n",
            "Epoch 18 Batch 700 Loss 1.2713 Accuracy 0.3365\n",
            "Epoch 18 Loss 1.2718 Accuracy 0.3365\n",
            "Time taken for 1 epoch: 55.578734397888184 secs\n",
            "\n",
            "Epoch 19 Batch 0 Loss 1.1901 Accuracy 0.3628\n",
            "Epoch 19 Batch 50 Loss 1.1427 Accuracy 0.3423\n",
            "Epoch 19 Batch 100 Loss 1.1387 Accuracy 0.3436\n",
            "Epoch 19 Batch 150 Loss 1.1543 Accuracy 0.3447\n",
            "Epoch 19 Batch 200 Loss 1.1642 Accuracy 0.3440\n",
            "Epoch 19 Batch 250 Loss 1.1725 Accuracy 0.3444\n",
            "Epoch 19 Batch 300 Loss 1.1779 Accuracy 0.3436\n",
            "Epoch 19 Batch 350 Loss 1.1842 Accuracy 0.3431\n",
            "Epoch 19 Batch 400 Loss 1.1902 Accuracy 0.3433\n",
            "Epoch 19 Batch 450 Loss 1.1971 Accuracy 0.3426\n",
            "Epoch 19 Batch 500 Loss 1.2032 Accuracy 0.3422\n",
            "Epoch 19 Batch 550 Loss 1.2078 Accuracy 0.3419\n",
            "Epoch 19 Batch 600 Loss 1.2119 Accuracy 0.3416\n",
            "Epoch 19 Batch 650 Loss 1.2179 Accuracy 0.3410\n",
            "Epoch 19 Batch 700 Loss 1.2228 Accuracy 0.3405\n",
            "Epoch 19 Loss 1.2229 Accuracy 0.3405\n",
            "Time taken for 1 epoch: 55.86971569061279 secs\n",
            "\n",
            "Epoch 20 Batch 0 Loss 1.1357 Accuracy 0.3281\n",
            "Epoch 20 Batch 50 Loss 1.0965 Accuracy 0.3475\n",
            "Epoch 20 Batch 100 Loss 1.1018 Accuracy 0.3502\n",
            "Epoch 20 Batch 150 Loss 1.1103 Accuracy 0.3483\n",
            "Epoch 20 Batch 200 Loss 1.1181 Accuracy 0.3479\n",
            "Epoch 20 Batch 250 Loss 1.1223 Accuracy 0.3477\n",
            "Epoch 20 Batch 300 Loss 1.1308 Accuracy 0.3462\n",
            "Epoch 20 Batch 350 Loss 1.1365 Accuracy 0.3468\n",
            "Epoch 20 Batch 400 Loss 1.1409 Accuracy 0.3464\n",
            "Epoch 20 Batch 450 Loss 1.1474 Accuracy 0.3462\n",
            "Epoch 20 Batch 500 Loss 1.1537 Accuracy 0.3455\n",
            "Epoch 20 Batch 550 Loss 1.1603 Accuracy 0.3456\n",
            "Epoch 20 Batch 600 Loss 1.1657 Accuracy 0.3451\n",
            "Epoch 20 Batch 650 Loss 1.1723 Accuracy 0.3445\n",
            "Epoch 20 Batch 700 Loss 1.1788 Accuracy 0.3437\n",
            "Saving checkpoint for epoch 20 at ./checkpoints/train/ckpt-4\n",
            "Epoch 20 Loss 1.1793 Accuracy 0.3437\n",
            "Time taken for 1 epoch: 56.41071534156799 secs\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RrXaC6e2KLMM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "7abd25ea-b318-49c7-f2fe-bdf5ff64c2b4"
      },
      "source": [
        "translator.translate(\"os meus vizinhos ouviram sobre esta ideia.\")\n",
        "print (\"Real translation: and my neighboring homes heard about this idea .\")"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input: os meus vizinhos ouviram sobre esta ideia.\n",
            "Predicted translation: my neighbors heard about this idea .\n",
            "Real translation: and my neighboring homes heard about this idea .\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "igi6pQJ9M8KP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "61e7f7e4-26dd-4ed6-8acc-b79485da8daf"
      },
      "source": [
        "translator.translate(\"vou então muito rapidamente partilhar convosco algumas histórias de algumas coisas mágicas que aconteceram.\")\n",
        "print (\"Real translation: so i 'll just share with you some stories very quickly of some magical things that have happened .\")"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input: vou então muito rapidamente partilhar convosco algumas histórias de algumas coisas mágicas que aconteceram.\n",
            "Predicted translation: so i 'm going to really quickly share with you some of some magic stories that happened to happen .\n",
            "Real translation: so i 'll just share with you some stories very quickly of some magical things that have happened .\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "znRqEFjXNBT8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 653
        },
        "outputId": "38394724-be7a-4c24-ab4a-af6394e2bee8"
      },
      "source": [
        "translator.translate(\"este é o primeiro livro que eu fiz.\", plot='decoder_layer4_block2')\n",
        "print (\"Real translation: this is the first book i've ever done.\")"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input: este é o primeiro livro que eu fiz.\n",
            "Predicted translation: this is the first book that i did it .\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABHgAAAJFCAYAAACmxkAKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdeZikZXno/+/dPTPMwMywCyrKJgqKIDgacQvBuESNJ27HiMlxQTESRc3xeCR6XPJTY+LRc0UTo4iCRmOCoAaNxKgRUDAq+6KgRnFHBUWGZbbu+/dHVU/XND1Dd9fzvG+/1d/PdfXV3dU9dz1d0/2dt5+ptyoyE0mSJEmSJHXXWNsLkCRJkiRJ0nDc4JEkSZIkSeo4N3gkSZIkSZI6zg0eSZIkSZKkjnODR5IkSZIkqePc4JEkSZIkSeo4N3gkSZIkSZI6zg0eSZIkSZKkjnODR5IkSZIkqePc4JEkSZIkSeq4ZW0vQItfRKwA7tt/97rM3NzmeiSNNpsjqUk2R1LT7I5qicxsew1axCLiWOBDwPVAAPcCnpuZF7S4LEkjyuZIapLNkdQ0u6Oa3ODRDkXEJcDxmXld//37Ah/LzAe3uzJJo8jmSGqSzZHUNLujmnwMHt2V5VPxAcjMbwPLW1yPpNFmcyQ1yeZIaprdUTUjs8ETPZ+KiMPaXsuIuSQiTouIY/sv7wcubntR0mJgd6qwOdJ22JwqbI60HTanGrujakbmFK2IeDzwQeCfMvN/tr2eUREROwF/Cjyyf9GXgfdk5sb2ViUtDnanPJsjbZ/NKc/mSNtnc+qwO6pplDZ4zgROB/4GuH9mbml5SZ0XEePANZl5aNtrkRYju1OWzZF2zOaUZXOkHbM55dkd1TYSp2hFxF7AAzLzXOALwB+0vKSRkJkTwHURce+21yItNnanPJsjbZ/NKc/mSNtnc+qwO6ptJDZ4gD8GPtZ/+3TghS2uZdTsDlwTEV+MiHOmXtpelNoVEU+NiNVtr6NldqcOm6M7sTmAzanF5uhObA5gc2qyO7qTUt0ZiVO0IuIq4AmZ+ZP++1cAT87MH7W7su6LiN+e7fLMPL/ptWhxiIiDgWuBl2Xme9teT1vsTh02RzPZnB6bU4fN0Uw2p8fm1GN3NFPJ7nR+gycidgOelZnvG7jsscCNmXlZeyuTRlNEvLn/5uMy86GtLqYldkdqjs2xOVKTbI7NkZpWsjudP0UrM28Grp5x2eeBndtZ0WiIiK/0X6+PiFsGXtZHxC1tr0/t6D8w3DOBvwJ+ExFHtrykVtid8myOZmNzemxOeTZHs7E5PTanDruj2ZTuTuc3ePrePcfLNEeZ+cj+6zWZuXbgZU1mrm17fWrNE4H/zMz19J4284SW19Mmu1OQzdF22JxpNqcgm6PtsDnTbE5hdkfbUbQ7y4osqSURcQzwcGDviPizgQ+tBcbbWdXoiYhHAodk5un9R9Rfk5nfb3tdasUJwDv7b38SeHNEvCozN7W4pkbZnfpsjgbYHJtTnc3RAJtjcxphdzSgaHe6fg+eFcBqehtVawZebgGe0eK6RkZEvAH438Ap/YtWAB9pb0VqS/987N0y8wKAzNwAnAUc1+rCmmd3KrI5mmJztrI5FdkcTbE5W9mcyuyOptTozig8yPI4cGZmPr3ttYyiiLgcOAq4NDOP6l92ZWYe0e7KpPbYnXpsjnRnNqcemyPdmc2py+6opk6fogWQmRMRcY+21zHCNmVmRkQCRMQubS9IzYuIo3f08cy8tKm1LAZ2pyqbI5szg82pyubI5sxgc6qzO6rWnc5v8PRdHhHnAB8Hbpu6MDM/0d6SRsaZEfE+YLeIeBHwAuD9La9JzXtH//VKYB1wBRDAEcDFwDEtratNdqcOmyOwObOxOXXYHIHNmY3NqcfuCCp1p/OnaAFExOmzXJyZ+YLGFzOCIuKxwOPofcN9rv80iVqCIuITwBsy86r++4cDb8zMJXdOtt2px+Zois2ZZnPqsTmaYnOm2Zy67I6mlO7OSGzwqL6IWMvAPb4y81ctLkctiYhrMvMBd3WZNCybI7A5ao7NEdgcNcvuCMp3ZyRO0YqIlfSeXuwB9O7iBIA7zMOLiBcDbwI2AJP0dpkTOKjNdak1V0bEaUw/0v9zgCtbXE9r7E4dNkcz2Jw+m1OHzdEMNqfP5tRjdzRD0e50/WnSp/wDsC/weOB8YD9g/TADI2KfiPhARJzbf//+EXHC0CvtnlcBh2fmAZl5UGYemJlDxyci9ouIT0bELyPiFxFxdkTsV2C9quv5wDXAy/sv3+xfthQV7Y7N2crmaJDNmeaxTh02R4NszjSbU0/x7ticTivanZE4RSsiLsvMo6aeXi4ilgNfzsyHDTHzXOB04LWZeWRELAMuy8wHllp3F0TEvwFPy8zbC8/9PPCP9P7xAPgj4DmZ+diS1yPVUro7NqfH5kiz81inDpsjzc7m1FOjOzZHU0biFC1gc//1zf0HJboBuNuQM/fKzDMj4hSAzNwSERNDzuyiU4CLIuJrwMapCzPz5CHn7p2Zgw/edkZEvGLImUTE/sAhmfmFiFgFLMvMof63QdMi4hHAG4H92fac4aV4l9LS3bE5PZ1qDtidmmzONjzWqcPmaCubsw2bU0+N7ticjirdnVHZ4Dk1InYHXgecA6wG/s+QM2+LiD3pnQ9JRDwM+M2QM7vofcB/AFfRO0e0lJsi4o+Aj/XffzZw0zADo/c0gycCewAH07sr6XuBxwwzV9v4APBK4BJgKf6DPKh0d2xOT2eaA3anATZnmsc6ddgcDbI502xOPTW6Y3O6q2h3RuUUrQMz8/t3ddk8Zx4NvBs4HLga2Bt4ZmZeMdRiO2bq7pkV5u5P7/Y9hl7kLwJOzswfDjHzcuChwNem1hwRVy21u33WFBFfy8zfansdi0Hp7ticni41pz/X7lRkc6Z5rFOHzdEgmzPN5tRTozs2p7tKd2dU7sFzNnD0jMvOAh48xMxrgN8G7kfvkc2vY3QelHo+zo2IE4FPs+1dCBf8NH4RMQ68NTOfUmB9gzZm5qaImLqeZfT/h0DFfCki3g58gm2/Hy5tb0mtKd0dm9PTpeaA3anN5kzzWKcOm6NBNmeazamnaHdsTucV7U6nN3gi4lB6T923a0Q8beBDaxl4Or8F+mpmHk0vRFPXdyl3Dt2oe3b/9SkDlw31NH6ZORER+0fEiszcNNTqtnV+RPw5sCoiHgucRC+cKmdqd3ndwGUJHNfCWlpRsTs2p6dLzQG7U5vN8VinNpujQTbH5jShaHdsTucV7U6nN3jo7f4+GdgN+P2By9cDL1rIwIjYF7gnvW/io+jtLkMvajsvfKndlJkHVhr9PeDCiDgHuG3g+t45xMzXACfQO5/1xcBngdOGWaS2lZm/0/YaFoGi3bE52+pYc8DuVGVzAI91qrI5GmRzAJtTXaXu2JyOKt2dUXkMnmMy86uFZj0XeB69HbRvMB2g9cAZmfmJEtez2EXEcZn5HzN27rca9naIiDdsZ+6bhpmruiJiH+CtwD0y8/ci4v7AMZn5gZaX1rhS3bE5PTZHs7E50zzWKcvmaDY2Z5rNKa9md2xOd5Xuzqhs8Pw18GbgDuDfgCOAV2bmR4aY+fTMPLvQEjsnIt6UmW+IiNNn+XBm5guGnH906fOZI+L7zHJO6LBPbdm/DWabO9Rt0EURcS5wOvDazDyyfx7uZUvxgdZKd8fmdK85/bl2pyKbM81jnbJszp3m2hxsziCbU17N7tic7irdna6fojXlcZn56oh4KnA98DTgAmDBAQL2i4i19HaW30/v3NDXZOa/D7vYLujHZww4NzPPrHAV7+jfXfMs4J8z8+oCMwfPW1wJPJPeU/oN6zMz5j4V+OmwQyPiUcBFmTkxcFmVOBe0V2aeGRGnAGTmlohYqk8jWro7Nqd7zQG7U5vNmeaxTkE2505sTo/NmWZzCqvcHZtDJ5sDpbuTmZ1/Aa7pvz4NeEL/7SuGnHlF//XjgU/Se7CxS9v+Wlu4bS+uOHtf4GTgQnrndb6uwnVcUmHmGL1wDDvnduB84G4Dly3q7zHgPGDPqXUCDwPOb3tdLd0WRbtjc7beDp1uTv967E65r9vmTN8WHuvUuV1tzuwzbU7anP5rm1P+tq3SHZvTveb011e0O6NyD55PR8S19O5C+JKI2BvYMOTMqXNDnwh8ODOviYjY0R8YUV+IiFcB/8y2D9i14KcPHZhxA/CuiPgS8Grg9fTuCrogETH4CPxj9Haca3yPHwLcrcCc64C303t0+hMy8yKmv+8Wqz8DzgEOjogLgb2BZ7S7pNaU7o7N6elMc8DuNMDmTPNYpw6bMzubY3NsTj1VumNzgO41Bwp3ZyQegwcgIvYAfpO9p4nbBVjT/yZf6LzT6T3a+4HAkcA4cF5mPrjIgjuif97lTJnDn3d5GPAs4OnATfQCd3Zm/mKImV8aeHcLvbuT/t/MvG6IpRIR6+mdIxr91zcAp+SQ5xBHxKWZeXREHELv6/8g8ILsPX3kotU/L/R+9G6P6zJzc8tLak3J7ticni41pz/X7lRmc6Z5rFOezdk61+b02ZxpNqeOGt2xOVvndq45ULY7nd/giYidgUMy84qBy+4NTGTmT4aYOwY8CPheZt4cEXsC98zMK4detIiIr9L7oTszM4c+37KLIuKyzDyq//ZqegF6WmYuynvW1fpZ66Iat4XNqcvm9HSpOzZnmsc63WNzemxON9mc7rE5PV1qDlT6nWIENniWA9cCR2Tmbf3L/h3488y8eIi5ATwHOCgz/6J/Q++bmV8fcr1V5vZnHwk8qv/ulwe/UYaYuRI4CXgkvd3VLwPvzcxh76JZXET82Y4+npnvXODcqb+zAzPz/yv5dzbLdd07M39Yem4JtX7WuqjGbWFzts7sTHPA7tRkc6Z5rLN1rs2xOdXYnGk2Z5vZS7o7NqeuGj9rYwXX14r+3Zc+Cfx32LrjtXeBEL8HOAZ4dv/99cDfLWRQRDwyIsZLz51xHS8HPkrv3MW7AR+JiJcNOxf4ML0HQHs38Lf9t/9hiHWe2X99VURcOfByVUQMu3u/DngJvbt+3hP4E3qPzr+m/7JQU39nx/ffH+rvLCJe3X/97oh41+AL8Koh1llVxZ+1zql0W9icni41B+xONTZnmsc6NmeAzanE5kyzOVuvY9F3x+b0dLE5UOlnLRfBI0cP+wIcClzQf/t1wMkFZk49ivVlA5ct6JHjgYcDp5aeO+M6rgR2GXh/F+DKAnO/OZfL5jHv7v3X+8/2MuRaL6B3bvDU+2umvi8Wy/dC/8/e1H/9CuC5M1+GXW/Nlxo/a119KX1b2JytczrTnP5cu1PxxebUvS261B2bs3W+zan4YnPq3hZdak5/xqLvjs3Z+mc72Zz+mov+rC3Kc9HmKzOvjZ77An/I9N3ohrG5vyucANF75PjJBa7vooi4vfTcGQKYGHh/on/ZsC6NiIdl5n8CRMRvAQveUczMn/Vf/6DA2mbaB9g08P6m/mXDKv139vOIuAfwfOBYyvw9NaLSz1onVbgtbE5Pl5oDdqcqmzPNYx2b02dzKrI502wO0IHu2JytOtkcKP+zNhIbPH0fAE4DrsrMXxeY9y56d5e6W0S8hd5Tlb1uocMy8/IacwecDnwtIj7Zf/8P6N0mw3owcFFETJ23eG/guoi4it6jvR8xn2Ex/Yjpd/pQf97aIdb6YeDrM26DM4aYN6X039nfA18EDgIuGbh86lHkh3rmju2JiH1ziGc+GFD6Z63LSt4WNqenS80Bu7NdNqeKpXysY3N6bM4OFOqOzZm2lJsDHeiOzdmqy82Bgj9rnX+Q5SnRewTqnwFPz8wvFJp5KPAYet8YX8zMby3yuUfTe7Au6D0I2GUFZu6/o49X3C1ekP5tMLXreUGJ26A/t/jfWUT8fWa+ZOjFzf36/jUzn1RgTvGfta4qfVvYnO41B+zODq7L5hS21I91bE6Pzdnh9Q3dHZszbak3pz93yXfH5uzw+hbdsc7IbPBIkiRJkiQtVZ1/Fi1JkiRJkqSlzg0eSZIkSZKkjhu5DZ6IONG5zq0107n153aR37/O7eLcLq215twu6tpt7Pevc7s41+ZM69Lfm3PrzXRuN+aO3AYPUCvGzu3e3C6t1bnd5vevc7s4t0trrTm3i7p2G/v969wuzrU507r09+bcejOd24G5o7jBI0mSJEmStKR04lm0VoytzFVja+b0uZtyAytiZfE1zGdurlox97mbb2PF8l3m9Llxx6a5z53HejcdsHzOcyduuZ3xtTvf5ect/97GOc8E2JwbWD6X9c7j+3UzG1nOTvNah3PrzF3Pr2/MzL2LL6KiFbFTrmRuP5vzuS12f8DmOX3erb/ezOrd5/6zefN37vrnEmDTxB2sGF8157m5ecucPm/OP8N9MTa3/1/YlHewIua23pyYmPP1L4afi7bndmmt8527gdvYlBuj+CIqWhErc9XY6jl97ryOdcbn/n95mybvYMXY3H7eDjzs5jnPvemmSfbcc27r+P5VczzeYyMr5vF9Ntfj3cXw/evcxTF35JsztipXLZvjz9s82rBpj7n/PUzccRvjq+Z2rDU2t8MRALZsuI1lK+c2d9mtczsug3keQ22Z+zHJXI91cnJyzjPne1xWZW6l39vmegwJ8/wdfj637yJo2fZ+v1pWfFUVrBpbwzG7PrXtZczZ5sMPqDJ3+dXXV5n7o3fcvfjMe/7hfxWfCZAb57dx1Lqo9G99BzZmB30hz/pB22uYr5Xswm+NP6743Kef/bPiMwH+5XFHV5k7ccMvqswd22Xum0xzNXHzb4rPVDd9Lb/Y9hLmbdXYah6285OLzx1bO7df4ObrI5/9RJW5xx9yXJW5kxs2VJlbhccOndPJ5ixbw8P3+u/F5/7wjw8uPhNg5Y11vn/3/vINVeZy46+Kj5y87Y7iM2uaz3+8zcfYqvIbVwCTt99eZW4tX5j8+Ky/X3mKliRJkiRJUse5wSNJkiRJktRxbvBIkiRJkiR1nBs8kiRJkiRJHecGjyRJkiRJUsctaIMnInaLiJMG3j82Ij6znc89LSLuv9AFSpLNkdQkmyOpaXZHUgkLvQfPbsBJd/lZQGa+MDO/ucDrkSSwOZKaZXMkNc3uSBraQjd43gYcHBGXR8Tb+5etjoizIuLaiPhoRARARJwXEesiYjwizoiIqyPiqoh4ZZGvQNJSYHMkNcnmSGqa3ZE0tGUL/HOvAQ7PzAdB7y6EwFHAA4CfAhcCjwC+MvBnHgTcMzMP7/+Z3RZ43ZKWHpsjqUk2R1LT7I6koZV8kOWvZ+aPM3MSuBw4YMbHvwccFBHvjognALfsaFhEnBgRF0fExZtyQ8FlShoRRZsD23ZnMxvLr1hSl1Vtjsc6kmZR7/eryTvqrFhSq0pu8Az+NjTBjHsHZeavgSOB84A/AU7b0bDMPDUz12XmuhWxsuAyJY2Ios3p/5mt3VnOTgWXKmkEVG2OxzqSZlHv96uxVYWXKmkxWOgpWuuBNfP5AxGxF7ApM8+OiOuAjyzwuiUtPTZHUpNsjqSm2R1JQ1vQBk9m3hQRF0bE1cC5wL/O4Y/dEzg9IqbuNXTKQq5b0tJjcyQ1yeZIaprdkVTCQu/BQ2YeP+Oi8wY+9tKBt48d+JyjF3p9kpY2myOpSTZHUtPsjqRhlXwMHkmSJEmSJLXADR5JkiRJkqSOc4NHkiRJkiSp49zgkSRJkiRJ6jg3eCRJkiRJkjpuwc+i1aScmGDi5pvbXsacve3D76sy97X3fWSVufv9jx8Vn5kTE8VndlJm2yvQAkUEYyuWF5/7qWMOKT4TYPI+u1WZm3ffvcrcb71oZfGZh73uB8VnAkz8/BdV5kqDcuUK8rADy8/9/k+LzwR4+Y+fUGVuHLRvnbnfub780JwsPxOPodSQLRNM3vyb4mP3+3yd39luPGrXKnOpcKwH8OvfO6z4zN2/8fPiMwEmvvfDKnOZrNOyydtuqzKXiDpzG/590HvwSJIkSZIkdZwbPJIkSZIkSR3nBo8kSZIkSVLHucEjSZIkSZLUcW7wSJIkSZIkdZwbPJIkSZIkSR3nBo8kSZIkSVLHVd/giYiLal+HJE2xOZKaZnckNcnmSNqe6hs8mfnw2tchSVNsjqSm2R1JTbI5kraniXvw3Np/ffeIuCAiLo+IqyPiUbWvW9LSY3MkNc3uSGqSzZG0PcsavK7jgc9l5lsiYhzYeUefHBEnAicCrNzxp0rSbObVHJjRndil8vIkjaCFH+us2LWB5UkaMQtvjsc50khqcoPnG8AHI2I58KnMvHxHn5yZpwKnAqyNPbKB9UkaLfNqDmzbnV3H9rQ7kuZr4cc6u9zD5kiarwU3x+McaTQ19ixamXkB8GjgJ8AZEfE/mrpuSUuPzZHUNLsjqUk2R9JMjW3wRMT+wM8z8/3AacDRTV23pKXH5khqmt2R1CSbI2mmJk/ROhb4XxGxGbgVcIdZUk3HYnMkNetY7I6k5hyLzZE0oPoGT2au7r/+EPCh2tcnaWmzOZKaZnckNcnmSNqexk7RkiRJkiRJUh1u8EiSJEmSJHWcGzySJEmSJEkd5waPJEmSJElSx7nBI0mSJEmS1HFNPk36gsXYGGOrVhWfO3nHHcVnArz20EdVmRsr6vx1Pffiq4vP/NC6w4vPBMhbb60yl8w6c9VZmcnkxo3F58aWLcVnAuSl36oyd3zXtVXmfugxXyo+860nPbT4TKkpG/cJvvOK5cXn3u+V48VnAtz4gj2rzN20z+oqc3e6Ze/iMyd+eWPxmQBjO+1UZe7kbbdVmUtEnbld0sHDyKR3rFPa+A03FZ8JsPdn6/y8sbZOczb84c3FZ05+/EfFZwLEWJ2f4VixssrcWnJiss7czZuqzN0e78EjSZIkSZLUcW7wSJIkSZIkdZwbPJIkSZIkSR3nBo8kSZIkSVLHucEjSZIkSZLUcW7wSJIkSZIkdZwbPJIkSZIkSR031AZPROwWEScNvH9sRHxm+GVJ0p3ZHElNsjmSmmZ3JA1j2Hvw7AacdJefJUll2BxJTbI5kppmdyQt2LAbPG8DDo6IyyPi7f3LVkfEWRFxbUR8NCICICIeHBHnR8QlEfG5iLj7kNctaemxOZKaZHMkNc3uSFqwZUP++dcAh2fmg6B3F0LgKOABwE+BC4FHRMTXgHcD/y0zfxkRzwLeArxge4Mj4kTgRICVscuQy5Q0Iqo1pz9vujvsXOtrkNQdjTVnfK9da30Nkrqlmd+vPM6RRtKwGzyz+Xpm/hggIi4HDgBuBg4HPt/fcB4HfrajIZl5KnAqwK7je2WFdUoaDUWaA9t2Z23sYXckzaZKc3Y66J42R9L2FP/9au3YnjZHGkE1Nng2Drw90b+OAK7JzGMqXJ+kpc3mSGqSzZHUNLsjaU6GfQye9cCaOXzedcDeEXEMQEQsj4gHDHndkpYemyOpSTZHUtPsjqQFG2qDJzNvAi6MiKsHHgRsts/bBDwD+KuIuAK4HHj4MNctaemxOZKaZHMkNc3uSBrG0KdoZebxMy46b+BjLx14+3Lg0cNen6SlzeZIapLNkdQ0uyNpoYY9RUuSJEmSJEktc4NHkiRJkiSp49zgkSRJkiRJ6jg3eCRJkiRJkjrODR5JkiRJkqSOG/pZtJqQk5NM3n5728uYs9y4sVNz/3DNr4vPPH39+uIzpcZllh+5ZUvxmTVN/Lp8HwAevbL8zLds3lR+qNSQna7fwCEnfLP43InNdZozVum4bMXYflXmfvcdexafufun71V8JsCu//iNKnM7p8K/wRqQWeV3iy03/Lz4zKp+/osqYy97yOXFZz5+4qjiM4FqP2tdO+YdFd6DR5IkSZIkqePc4JEkSZIkSeo4N3gkSZIkSZI6zg0eSZIkSZKkjnODR5IkSZIkqePc4JEkSZIkSeo4N3gkSZIkSZI6bt4bPBFxckR8KyI+GhFPiYjXzOPPHhARx8/3OiUtbXZHUpNsjqQm2RxJpSxbwJ85CfjdzPxx//1zZn5CRCzLzC2z/NkDgOOBf1zA9UpauuyOpCbZHElNsjmSipjXBk9EvBc4CDg3Ij4I/BpYl5kvjYgzgA3AUcCFEfEvwN/0/2gCjwbeBhwWEZcDH8rM/1fmy5A0quyOpCbZHElNsjmSSprXBk9m/klEPAH4ncy8MSKeN+NT9gMenpkTEfFp4E8z88KIWE0vTq8BXpWZT76r64qIE4ETAVay83yWKWmE2B1JTbI5kppkcySVVPpBlj+emRP9ty8E3hkRJwO7becuhduVmadm5rrMXLecnQovU9IIsTuSmlSnObGy+EIljQSPcyTNWekNntum3sjMtwEvBFbRu0vhoYWvS5LA7khqls2R1CSbI2nOFvIgy3MSEQdn5lXAVRHxEOBQ4EfAmlrXKWlpszuSmmRzJDXJ5ki6K6XvwTPoFRFxdURcCWwGzgWuBCYi4oqIeGXF65a0NNkdSU2yOZKaZHMk7dC878GTmQcMvH0GcEb/7efN+LyXbWfEcfO9TklLm92R1CSbI6lJNkdSKTXvwSNJkiRJkqQGuMEjSZIkSZLUcW7wSJIkSZIkdZwbPJIkSZIkSR3nBo8kSZIkSVLHzftZtFoRQey0U/GxuWlT8ZkAY6tXV5k7edvtVeY+8YHlH3h/+Xnl/74AJl66a5W5ee1368ydzCpzmZyoM1fbGhsvPzMny88EyDrfa7F8RZW5T3rok4rPPOHbXyk+E+ADh96nylx/jrWthIkK3xOVvs8m79hQZS7fqvPv8cEv3aP4zM9e8fniMwGe8E/rqsyt9c9PrX9/VFlElX/jx3ZdU3wmAOMVjsmAiV/eVGXukx7+lOIzv3PGXsVnAhz6sjrdrfV3VuXfSoAVy6uMnbjpV1Xmsp30eg8eSZIkSZKkjnODR5IkSZIkqePc4JEkSZIkSeo4N3gkSZIkSZI6zg0eSZIkSZKkjnODR5IkSZIkqeN2uMETEQdExNUlrigiro+IOs/tJmkk2BxJTbM7kppkcyTV5D14JEmSJEmSOm4uGzzLIuKjEfGtiDgrInYGiIjHRMRlEXFVRHwwInba0eVTImJVRJwbES+q8PVI6j6bI6lpdkdSk2yOpCrmsmVf1doAACAASURBVMFzP+A9mXkYcAtwUkSsBM4AnpWZDwSWAS/Z3uUDs1YDnwY+lpnvL/ZVSBolNkdS0+yOpCbZHElVzGWD50eZeWH/7Y8Aj6QXpe9n5rf7l38IePQOLp/yL8Dpmfnhu7rSiDgxIi6OiIs354Y5LFPSiGilOTCjO2wc9uuQ1B2L4FjH5khLyCJojr9fSaNoLhs8eRfvz8eFwBMiIu7ySjNPzcx1mblueawc4ioldUwrzYEZ3WGnu/4DkkbFIjjWsTnSErIImuPvV9IomssGz70j4pj+28cDXwGuAw6IiPv0L/9j4PwdXD7l9cCvgb8bduGSRpbNkdQ0uyOpSTZHUhVz2eC5DvjTiPgWsDvw95m5AXg+8PGIuAqYBN67vctnzHs5sCoi/rrUFyFppNgcSU2zO5KaZHMkVbFsRx/MzOuBQ7fzsS8CR83j8gMG3n3+fBYpaWmwOZKaZnckNcnmSKppLvfgkSRJkiRJ0iLmBo8kSZIkSVLHucEjSZIkSZLUcW7wSJIkSZIkdZwbPJIkSZIkSR23w2fRWjQyyU2bqsytYXL9+ipzq4koPnLzcb8oPhNgfPeNVebe8OKHVpm78y8mqsxd9cvNVeauuPoHVebyyzpjuyjGx6vMzS1bKs2t872Wu6wqPvOMdUcWnwmw7IA9qsydXL2yylz+60dVxn77LQ+sMvd+b/xW8ZlxS52fs7oCokP/7zZZ59+3WiZu+lXxmU865veLzwTYdNw+VeauvOR7VeZufNCBVeb+7GE7VZl74Md+Wnxm/HhF8ZnVZZKby/9+NXHjTcVndtGW639YfOYhzy0/E6BWzce/dI8qc3/znntVmbv2X6+qMrfWnsP2dOhIQpIkSZIkSbNxg0eSJEmSJKnj3OCRJEmSJEnqODd4JEmSJEmSOs4NHkmSJEmSpI5zg0eSJEmSJKnj3OCRJEmSJEnquHlt8ETEbhFx0sD7x0bEZ+Y543kRcY/5/BlJS5PNkdQ0uyOpSTZHUknzvQfPbsBJd/lZO/Y8wABJmgubI6lpdkdSk2yOpGLmu8HzNuDgiLg8It7ev2x1RJwVEddGxEcjIgAi4vUR8Y2IuDoiTo2eZwDrgI/2Z6wq+LVIGj02R1LT7I6kJtkcScXMd4PnNcB/ZeaDMvN/9S87CngFcH/gIOAR/cv/NjMfkpmHA6uAJ2fmWcDFwHP6M+4Y/kuQNMJsjqSm2R1JTbI5koop8SDLX8/MH2fmJHA5cED/8t+JiK9FxFXAccAD5jM0Ik6MiIsj4uLNbCywTEkjokpzwO5I2q76xzq5oeyKJXWZv19JWpBlBWYM1mECWBYRK4H3AOsy80cR8UZg5XyGZuapwKkAa2OPLLBOSaOhSnPA7kjarvrHOmN72hxJU/z9StKCzPcePOuBNXP4vKnY3BgRq4FnLGCGJNkcSU2zO5KaZHMkFTOvDZ7MvAm4sP/AXm/fwefdDLwfuBr4HPCNgQ+fAbzXBwGTdFdsjqSm2R1JTbI5kkqa9ylamXn8jIvOG/jYSwfefh3wuln+/NnA2fO9XklLk82R1DS7I6lJNkdSKSUeZFmSJEmSJEktcoNHkiRJkiSp49zgkSRJkiRJ6jg3eCRJkiRJkjrODR5JkiRJkqSOm/ezaLUms+0VjKzcuKntJcxZ3nZ7lbk3H76lytxdvhBV5q746W+qzJ1cf2uVuZ01OVF8ZE4WH1lXrfbe+KviIyduva34TIC4vU53vvuXD6ky977v21hl7u5X1+nZxC3lu5MT5X92q8skN3fn32OizvdDteZUiG/+us6/xSsvrdMcJuvctj/+nRVV5q64pcpYctz/35aacOPtu1SZO7FHnZ/h1XfcUWVu0yycJEmSJElSx7nBI0mSJEmS1HFu8EiSJEmSJHWcGzySJEmSJEkd5waPJEmSJElSx7nBI0mSJEmS1HFu8EiSJEmSJHVcKxs8EXFRG9craemyO5KaZHMkNcnmSIKWNngy8+FtXK+kpcvuSGqSzZHUJJsjCdq7B8+tbVyvpKXL7khqks2R1CSbIwl8DB5JkiRJkqTOW9b2ArYnIk4ETgRYyc4tr0bSUmB3JDXJ5khqks2RRt+ivQdPZp6amesyc91ydmp7OZKWALsjqUk2R1KTbI40+hbtBo8kSZIkSZLmxg0eSZIkSZKkjmvradJXt3G9kpYuuyOpSTZHUpNsjiTwHjySJEmSJEmd5waPJEmSJElSx7nBI0mSJEmS1HFu8EiSJEmSJHWcGzySJEmSJEkdt6ztBczZ2Hj5mZMT5Wd20OStt5Yfmll+JjC5oc7f2f3/6oYqc29498oqc3f+yzpPlLD8lt2qzOVndcZWZ3eqmfjVr8sPrdSdWLVLlbmH/EOF2wD47gv3rTJ3j6vq3L7je+5RfGb8qjuHN51V6eetmgrrnbjlluIzAWJZne/fWnO/+fy/qzL39w9/TJW5uWlT+aGbt5SfWVksX8ayvfYpPnfLDT8vPrOTIsrPrHWcs3xFlbl7/dGNVeZ+6y93rzJ3l//2kCpz13zth1Xm8tPZL/YePJIkSZIkSR3nBo8kSZIkSVLHucEjSZIkSZLUcW7wSJIkSZIkdZwbPJIkSZIkSR3nBo8kSZIkSVLHucEjSZIkSZLUccU3eCLijRHxqv7bfxERvzvL5xwbEZ8pfd2Slh6bI6lJNkdSk2yOpPlYVnN4Zr6+5nxJGmRzJDXJ5khqks2RdFeK3IMnIl4bEd+OiK8A9xu4/IyIeEb/7SdExLURcSnwtBLXK2lpsjmSmmRzJDXJ5khaqKE3eCLiwcAfAg8Cngg8ZJbPWQm8H/h94MHAvsNer6SlyeZIapLNkdQkmyNpGCXuwfMo4JOZeXtm3gKcM8vnHAp8PzO/k5kJfOSuhkbEiRFxcURcvJmNBZYpaURUaQ7YHUmzsjmSmtRIczZN3lFwyZIWi0X7LFqZeWpmrsvMdcvZqe3lSFoC7I6kJtkcSU0abM6KsVVtL0dSBSU2eC4A/iAiVkXEGnp3FZzpWuCAiDi4//6zC1yvpKXJ5khqks2R1CSbI2nBhn4Wrcy8NCL+GbgC+AXwjVk+Z0NEnAj8a0TcDnwZWDPsdUtaemyOpCbZHElNsjmShlHkadIz8y3AW2a5/HkDb/8bvfNFJWkoNkdSk2yOpCbZHEkLtWgfg0eSJEmSJElz4waPJEmSJElSx7nBI0mSJEmS1HFu8EiSJEmSJHWcGzySJEmSJEkdV+RZtNRt43vuUXzmxK9uLj4TgJysMnZi712rzF328dVV5t70wCpj2ec3u9UZ/LM6Y6ubnGh7BSNr2T3vUXzmlp/8tPhMgMk7NlSZu+yW26vMPeR9d1SZ+80/v1uVubt9e9/iM/PW5cVnNiKi/MzM8jPVTePjVcY+Zd0Tq8z9xdMOrDJ32dN/WXzmlpNtTmdVug2W7X+v4jMnfnJD8ZkAMV7nPh+x86oqcw97x41V5v7kSeWPRwDIe9eZ+6nZL/YePJIkSZIkSR3nBo8kSZIkSVLHucEjSZIkSZLUcW7wSJIkSZIkdZwbPJIkSZIkSR3nBo8kSZIkSVLHucEjSZIkSZLUca1s8ETERf3XB0TE8W2sQdLSYXMkNcnmSGqSzZE0pZUNnsx8eP/NAwAjJKkqmyOpSTZHUpNsjqQpbd2D59b+m28DHhURl0fEK9tYi6TRZ3MkNcnmSGqSzZE0ZVnL1/8a4FWZ+eSZH4iIE4ETAVayc9PrkjSattscsDuSirM5kpo09+aMr25yXZIasmgfZDkzT83MdZm5bjk7tb0cSUuA3ZHUJJsjqUmDzVkxtqrt5UiqYNFu8EiSJEmSJGlu2t7gWQ+saXkNkpYOmyOpSTZHUpNsjrTEtb3BcyUwERFX+EBgkhpgcyQ1yeZIapLNkZa4Vh5kOTNX919vBo5rYw2Slg6bI6lJNkdSk2yOpClt34NHkiRJkiRJQ3KDR5IkSZIkqePc4JEkSZIkSeo4N3gkSZIkSZI6zg0eSZIkSZKkjmvlWbTmK5YvY9neexefu+Xnvyw+s6qcrDJ24qZfVZlbw/iaNXUGf+eHVcbuefu+Vebe8Og9qsz9+SPqzOXqOmNrirExxnbepfjcydtvLz4TIMbHq8zNLVuqzN3y45+UHxpRfiYwtmJ5lblsrnPbTt6yvspcJvapMvbW/XcuPnPyO937/6sYH2d87dricydu/k3xmaorJybqzK307w+V1rtp1zpNz4nyfcgsPrK6jXvtxPdPOKj43Hu9+YbiM6uq9Je35foKv1tUOs6BOsc5W37y0ypzY/mKKnPv+fk6t8MPnrJnlbl8avaLu3cEJEmSJEmSpG24wSNJkiRJktRxbvBIkiRJkiR1nBs8kiRJkiRJHecGjyRJkiRJUse5wSNJkiRJktRxbvBIkiRJkiR1nBs8kiRJkiRJHecGjyRJkiRJUse5wSNJkiRJktRxy9pewPZExInAiQArx1e3vBpJS8E23YldWl6NpFG3TXPGbI6kugabs2zX3VtejaQaFu09eDLz1Mxcl5nrVoytans5kpaAbboTK9tejqQRt21zPNaRVNdgc8Z3dlNZGkWLdoNHkiRJkiRJc+MGjyRJkiRJUse1vsETEadFxLq21yFpabA5kppmdyQ1yeZIS1frD7KcmS9sew2Slg6bI6lpdkdSk2yOtHS1fg8eSZIkSZIkDccNHkmSJEmSpI5zg0eSJEmSJKnj3OCRJEmSJEnqODd4JEmSJEmSOi4ys+013KWI+CXwgzl++l7AjRWW4dzuze3SWkd97v6ZuXeFNVSzCLqzGP7enDvac7u01vnOtTkL06W5XVqrc7s51+ZM69Lfm3PrzXTu4po7a3c6scEzHxFxcWauc65zu7RW53ab37/O7eLcLq215twu6tpt7Pevc7s41+ZM69Lfm3PrzXRuN+Z6ipYkSZIkSVLHucEjSZIkSZLUcaO4wXOqc51bcaZz68/tIr9/ndvFuV1aa825XdS129jvX+d2ca7Nmdalvzfn1pvp3A7MHbnH4FHzIuLWzFw98P7zgHWZ+dICs88DXpWZF8+4/KXAK4CDgb0zs8aDXElahFpqzkeBdcBm4OvAizNz87DXJ2nxa6k5H6DXnAC+DTwvM28d9vokdUMb3Rn4+LuAFwxev7pjFO/Bo6XhQuB3mfuj/0vSMD4KHAo8EFgFvLDd5Ugaca/MzCMz8wjgh8DQv9RJ0l2JiHXA7m2vQwvnBo+qioi9I+LsiPhG/+UR/csfGhFfjYjLIuKiiLhf//JVEfFPEfGtiPgkvV+k7iQzL8vM65v7SiR1QcXmfDb76N2DZ7/GvihJi1bF5tzS//zof453uZcE1OtORIwDbwde3dgXo+KWtb0AjYRVEXH5wPt7AOf03/4b4P9l5lci4t7A54DDgGuBR2Xmloj4XeCtwNOBlwC3Z+ZhEXEEcGljX4WkrmitORGxHPhj4OVFvyJJi1krzYmI04EnAt8E/mfpL0rSotZGd14KnJOZP+vtLauL3OBRCXdk5oOm3pk6R7T/7u8C9x+IxNqIWA3sCnwoIg6h979Sy/sffzTwLoDMvDIirqy/fEkd02Zz3gNckJlfLvGFSOqEVpqTmc/v/4/6u4FnAacX+4okLXaNdici7gE8Ezi2+FeiRrnBo9rGgIdl5obBCyPib4EvZeZTI+IA4LzmlyZpBFVrTkS8AdgbePHwy5Q0Iqoe52TmRET8E71TJtzgkQR1unMUcB/gu/2No50j4ruZeZ8iK1ZjfAwe1fbvwMum3omIqZ3oXYGf9N9+3sDnXwAc3//cw4Ej6i9R0gip0pyIeCHweODZmTlZdsmSOqx4c6LnPlNvA0+hd+qFJEGF7mTmv2bmvpl5QGYeQO+ULjd3OsgNHtV2MrAuIq6MiG8Cf9K//K+Bv4yIy9j2nmR/D6yOiG8BfwFcMtvQiDg5In5M74FOr4yI06p9BZK6pEpzgPcC+wBfjYjLI+L1dZYvqWNqNCfonWZxFXAVcPf+50oS1DvW0QiI3hOCSJIkSZIkqau8B48kSZIkSVLHucEjSZIkSZLUcW7wSJIkSZIkdZxPk667FBErgPv2370uMze3uR5Jo83mSGqSzZHUNLujWnyQZe1QRBwLfAi4nt6zOtwLeG5mXtDisiSNKJsjqUk2R1LT7I5qcoNHOxQRlwDHZ+Z1/ffvC3wsMx/c7sokjSKbI6lJNkdS0+yOavIxeHRXlk/FByAzvw0sb3E9kkabzZHUJJsjqWl2R9WMzAZP9HwqIg5rey0j5pKIOC0iju2/vB+4uO1FSYuB3anC5kjbYXOqsDnSdticauyOqhmZU7Qi4vHAB4F/ysz/2fZ6RkVE7AT8KfDI/kVfBt6TmRvbW5W0ONid8myOtH02pzybI22fzanD7qimUdrgORM4Hfgb4P6ZuaXlJXVeRIwD12TmoW2vRVqM7E5ZNkfaMZtTls2RdszmlGd3VNtInKIVEXsBD8jMc4EvAH/Q8pJGQmZOANdFxL3bXou02Nid8myOtH02pzybI22fzanD7qi2kdjgAf4Y+Fj/7dOBF7a4llGzO3BNRHwxIs6Zeml7UWpXRDw1Ila3vY6W2Z06bI7uxOYANqcWm6M7sTmAzanJ7uhOSnVnJE7RioirgCdk5k/6718BPDkzf9TuyrovIn57tssz8/ym16LFISIOBq4FXpaZ7217PW2xO3XYHM1kc3psTh02RzPZnB6bU4/d0Uwlu9P5DZ6I2A14Vma+b+CyxwI3ZuZl7a1MGk0R8eb+m4/LzIe2upiW2B2pOTbH5khNsjk2R2paye50/hStzLwZuHrGZZ8Hdm5nRaMhIr7Sf70+Im4ZeFkfEbe0vT61o//AcM8E/gr4TUQc2fKSWmF3yrM5mo3N6bE55dkczcbm9NicOuyOZlO6O53f4Ol79xwv0xxl5iP7r9dk5tqBlzWZubbt9ak1TwT+MzPX03vazBNaXk+b7E5BNkfbYXOm2ZyCbI62w+ZMszmF2R1tR9HuLCuypJZExDHAw4G9I+LPBj60FhhvZ1WjJyIeCRySmaf3H1F/TWZ+v+11qRUnAO/sv/1J4M0R8arM3NTimhpld+qzORpgc2xOdTZHA2yOzWmE3dGAot3p+j14VgCr6W1UrRl4uQV4RovrGhkR8QbgfwOn9C9aAXykvRWpLf3zsXfLzAsAMnMDcBZwXKsLa57dqcjmaIrN2crmVGRzNMXmbGVzKrM7mlKjO6PwIMvjwJmZ+fS21zKKIuJy4Cjg0sw8qn/ZlZl5RLsrk9pjd+qxOdKd2Zx6bI50ZzanLrujmjp9ihZAZk5ExD3aXscI25SZGREJEBG7tL0gNS8ijt7RxzPz0qbWshjYnapsjmzODDanKpsjmzODzanO7qhadzq/wdN3eUScA3wcuG3qwsz8RHtLGhlnRsT7gN0i4kXAC4D3t7wmNe8d/dcrgXXAFUAARwAXA8e0tK422Z06bI7A5szG5tRhcwQ2ZzY2px67I6jUnc6fogUQEafPcnFm5gsaX8wIiojHAo+j9w33uf7TJGoJiohPAG/IzKv67x8OvDEzl9w52XanHpujKTZnms2px+Zois2ZZnPqsjuaUro7I7HBo/oiYi0D9/jKzF+1uBy1JCKuycwH3NVl0rBsjsDmqDk2R2Bz1Cy7IyjfnZE4RSsiVtJ7erEH0LuLEwDuMA8vIl4MvAnYAEzS22VO4KA216XWXBkRpzH9SP/PAa5scT2tsTt12BzNYHP6bE4dNkcz2Jw+m1OP3dEMRbvT9adJn/IPwL7A44Hzgf2A9cMMjIh9IuIDEXFu//37R8QJQ6+0e14FHJ6ZB2TmQZl5YGYOHZ+I2C8iPhkRv4yIX0TE2RGxX4H1qq7nA9cAL++/fLN/2VJUtDs2Zyubo0E2Z5rHOnXYHA2yOdNsTj3Fu2NzOq1od0biFK2IuCwzj5p6ermIWA58OTMfNsTMc4HTgddm5pERsQy4LDMfWGrdXRAR/wY8LTNvLzz388A/0vvHA+CPgOdk5mNLXo9US+nu2JwemyPNzmOdOmyONDubU0+N7tgcTRmJU7SAzf3XN/cflOgG4G5DztwrM8+MiFMAMnNLREwMObOLTgEuioivARunLszMk4ecu3dmDj542xkR8YohZxIR+wOHZOYXImIVsCwzh/rfBk2LiEcAbwT2Z9tzhpfiXUpLd8fm9HSqOWB3arI52/BYpw6bo61szjZsTj01umNzOqp0d0Zlg+fUiNgdeB1wDrAa+D9DzrwtIvakdz4kEfEw4DdDzuyi9wH/AVxF7xzRUm6KiD8CPtZ//9nATcMMjN7TDJ4I7AEcTO+upO8FHjPMXG3jA8ArgUuApfgP8qDS3bE5PZ1pDtidBticaR7r1GFzNMjmTLM59dTojs3prqLdGZVTtA7MzO/f1WXznHk08G7gcOBqYG/gmZl5xVCL7Zipu2dWmLs/vdv3GHqRvwg4OTN/OMTMy4GHAl+bWnNEXLXU7vZZU0R8LTN/q+11LAalu2NzerrUnP5cu1ORzZnmsU4dNkeDbM40m1NPje7YnO4q3Z1RuQfP2cDRMy47C3jwEDOvAX4buB+9Rza/jtF5UOr5ODciTgQ+zbZ3IVzw0/hFxDjw1sx8SoH1DdqYmZsiYup6ltH/HwIV86WIeDvwCbb9fri0vSW1pnR3bE5Pl5oDdqc2mzPNY506bI4G2ZxpNqeeot2xOZ1XtDud3uCJiEPpPXXfrhHxtIEPrWXg6fwW6KuZeTS9EE1d36XcOXSj7tn916cMXDbU0/hl5kRE7B8RKzJz01Cr29b5EfHnwKqIeCxwEr1wqpyp3eV1A5clcFwLa2lFxe7YnJ4uNQfsTm02x2Od2myOBtkcm9OEot2xOZ1XtDud3uCht/v7ZGA34PcHLl8PvGghAyNiX+Ce9L6Jj6K3uwy9qO288KV2U2YeWGn094ALI+Ic4LaB63vnEDNfA5xA73zWFwOfBU4bZpHaVmb+TttrWASKdsfmbKtjzQG7U5XNATzWqcrmaJDNAWxOdZW6Y3M6qnR3RuUxeI7JzK8WmvVc4Hn0dtC+wXSA1gNnZOYnSlzPYhcRx2Xmf8zYud9q2NshIt6wnblvGmau6oqIfYC3AvfIzN+LiPsDx2TmB1peWuNKdcfm9NgczcbmTPNYpyybo9nYnGk2p7ya3bE53VW6O6OywfPXwJuBO4B/A44AXpmZHxli5tMz8+xCS+yciHhTZr4hIk6f5cOZmS8Ycv7Rpc9njojvM8s5ocM+tWX/Npht7lC3QRdFxLnA6cBrM/PI/nm4ly3FB1or3R2b073m9OfanYpszjSPdcqyOXeaa3OwOYNsTnk1u2Nzuqt0d7p+itaUx2XmqyPiqcD1wNOAC4AFBwjYLyLW0ttZfj+9c0Nfk5n/Puxiu6AfnzHg3Mw8s8JVvKN/d82zgH/OzKsLzBw8b3El8Ex6T+k3rM/MmPtU4KfDDo2IRwEXZebEwGVV4lzQXpl5ZkScApCZWyJiqT6NaOnu2JzuNQfsTm02Z5rHOgXZnDuxOT02Z5rNKaxyd2wOnWwOlO5OZnb+Bbim//o04An9t68YcuYV/dePBz5J78HGLm37a23htr244ux9gZOBC+md1/m6CtdxSYWZY/TCMeyc24HzgbsNXLaov8eA84A9p9YJPAw4v+11tXRbFO2Ozdl6O3S6Of3rsTvlvm6bM31beKxT53a1ObPPtDlpc/qvbU7527ZKd2xO95rTX1/R7ozKPXg+HRHX0rsL4UsiYm9gw5Azp84NfSLw4cy8JiJiR39gRH0hIl4F/DPbPmDXgp8+dGDGDcC7IuJLwKuB19O7K+iCRMTgI/CP0dtxrvE9fghwtwJzrgPeTu/R6U/IzIuY/r5brP4MOAc4OCIuBPYGntHuklpTujs2p6czzQG70wCbM81jnTpszuxsjs2xOfVU6Y7NAbrXHCjcnZF4DB6AiNgD+E32niZuF2BN/5t8ofNOp/do7wcCRwLjwHmZ+eAiC+6I/nmXM2UOf97lYcCzgKcDN9EL3NmZ+YshZn5p4N0t9O5O+n8z87ohlkpErKd3jmj0X98AnJJDnkMcEZdm5tERcQi9r/+DwAuy9/SRi1b/vND70bs9rsvMzS0vqTUlu2NzerrUnP5cu1OZzZnmsU55NmfrXJvTZ3Om2Zw6anTH5myd27nmQNnudH6DJyJ2Bg7JzCsGLrs3MJGZPxli7hjwIOB7mXlzROwJ3DMzrxx60SIivkrvh+7MzBz6fMsuiojLMvOo/tur6QXoaZm5KO9ZV+tnrYtq3BY2py6b09Ol7ticaR7rdI/N6bE53WRzusfm9HSpOVDpd4oR2OBZDlwLHJGZt/Uv+3fgzzPz4iHmBvAc4KDM/Iv+Db1vZn59yPVWmfv/t3fnUZae9Z3Yv7/qXWqBAMGwDcgCDGIxYBpswIDYwmKHMQPHNspxRiR2HyMDhgRsJiFk4oQMMSeejPcjCAhieZkBY7MMg2c4ltkMSAYhAULYLDZil4xHC/RW9eSPui2Vmu5WLe/z3n6rPp9z6vS9t25936fu8q23fv3WvbPshyd5wuzsB1c+UDaQuTvJhUl+LMvT1Q8m+b3W2kYP0RxcVf0PJ/t8a+3X15l79D77gdba/z7kfXacbd2ntfb3Q+cOoddzbYp63BY655bMyXROond60jm3sq9zS67O0Tnd6Jxb6ZzbZG/p3tE5ffV4ri0MuL65mB2+9I4kP5XcMvG66wBF/DtJHpvkBbPzNyb57fUEVdWPVdW2oXOP2cYvJbkky3+7eLckv19VL9lobpK3ZvkF0H4zyW/NTv9/G1jnv5v9e1VVXbni46qq2uj0fl+SF2X50M97JfmFLL86/xmzj/U6ep+dPzu/ofusqn559u9vVtVvrPxI8ooNrLOrjs+1yel0W+icZVPqnETvdKNzbmVfR+esoHM60Tm30jm3bOOU7x2ds2yKnZN0jOCJsgAAIABJREFUeq61U+CVozf6keRBST4wO/3qJC8dIPPoq1h/csVl63rl+CSPS3LR0LnHbOPKJKevOH96kisHyP3sai5bQ949Zv/e93gfG1zrB7L8t8FHz59x9HFxqjwWZl97/ezflyX5F8d+bHS9PT96PNem+jH0baFzbsmZTOfMcvVOxw+d0/e2mFLv6Jxb8nVOxw+d0/e2mFLnzDJO+d7RObd87SQ7Z7bmQZ9rp+Tfoq1Va+1ztewHk/xMbj2MbiMOz6bCLUlq+ZXjl9a5vo9U1XeHzj1GJVlccX5xdtlGfaKqfrS19tEkqaofSbLuiWJr7euzf/9ugLUd658kObTi/KHZZRs19H32zaq6Z5IXJjkvw9xPo+j0XJukDreFzlk2pc5J9E5XOudW9nV0zozO6Ujn3ErnJJlA7+icW0yyc5Lhn2ubYsAz8/8meWOSq1pr3xkg7zeyfLjU3arqtVl+q7JXrzestXZFj9wV3pzkY1X1jtn5n8zybbJRj0rykao6+neL90lyTVVdleVXe/+htYTVra+Y/n2fmuXdYQNrfWuSjx9zG1y8gbyjhr7PfjfJ+5Ock+SvV1x+9FXkN/TOHSdSVXdvG3jngxWGfq5N2ZC3hc5ZNqXOSfTOCemcLrbyvo7OWaZzTmKg3tE5t9rKnZNMoHd0zi2m3DnJgM+1yb/I8lG1/ArUX0/yvNbafx4o80FJnprlB8b7W2tXn+K5P5zlF+tKll8E7JMDZN73ZJ/vOC1el9ltcHTq+YEhboNZ7uD3WVX9bmvtRRte3Oq3957W2o8PkDP4c22qhr4tdM70OifROyfZls4Z2Fbf19E5y3TOSbe34d7RObfa6p0zy93yvaNzTrq9U25fZ9MMeAAAAAC2qsm/ixYAAADAVrfpBjxVtV+u3F6ZcvvnTpHHr9wp5k5prT1zp2hqt7HHr9wp5uqcW03pfpPbL1PuNHI33YAnSa8ylju93CmtVe60efzKnWLulNbaM3eKpnYbe/zKnWKuzrnVlO43uf0y5U4gdzMOeAAAAAC2lEm8yPLOhT1tz/YzVnXdQ0vfy86FPau67oF77lz1GhZvvDnbzjh9Vdfdfe3BVeceat/LzlrdepdO27Xq3MOHb86OHatb78LNB1ade6gdyM7afbvXW+vj6nA7kB2ryK2dq7/PDi1+Nzu3nbaq67ZDq7/PDreD2VGrvC/WcDMczsHsyOrv4y65tYbcNdwON7bvXNdau+vq0+dvZ+1qu7O659BabuN7P+ymVV3vH/9hKWfeefUz+K9efcdVXe/Q0oHsXLj959otFpdWl5uD2bmGx+9qO+KUeF5s0twprXWtuQdycw61g2totPnr1Tk/8LAbV72G6/9hKXdZZe98+TOr65xk9fsOSdLa6jpnTT+Lk6z2B9xq90eSJGvY1zkVnhdy+2XqnFvd/4duXvUarrt+MWfdZduqrvuFz6z+3b7X8vtVW1pcde6UfgeY0nNN7vpyb8zxf7/aPviqOtiz/Yw87q4/PXjuZ1/9TwfPTJJzf/nzXXIP/vD9u+Tu/NjnBs9sR44MnpkkC/e9d5fcpS/2eTfCXrdDqs8+RG1b3Q/ZtfpPh//olHq7x9XYndPzI/XUwXNf/+6PDp6ZJK/at+F3aDyudtPqd9TWYunA6gfLsFYfa++f9xLWbHdOz48sPG3w3Iv/wwcHz0yS/+5B/1WX3HboUJfcdPj51g6u/j+H2Nwm2zkd9nPe8d6PD56ZJM9/6NO75C512s/p9jsAzPzn9rbj/n7lT7QAAAAAJs6ABwAAAGDiDHgAAAAAJs6ABwAAAGDiDHgAAAAAJs6ABwAAAGDi1jXgqaozq+rCFefPq6p3n+C6b6yqB693gQA6BxiTzgHGpneAIaz3CJ4zk1x4u9dK0lr7udbaZ9e5HYBE5wDj0jnA2PQOsGHrHfC8Lsn9quqKqnr97LK9VfW2qvpcVV1SVZUkVXVpVe2rqm1VdXFVfbqqrqqqlw/yHQBbgc4BxqRzgLHpHWDDtq/z616V5KGttUcky4cQJnlkkock+VqSDyd5fJIPrfiaRyS5V2vtobOvOfNkG6iq/Un2J8nubXvXuUxgk+jeObPr3No7OW3A5QMTo3OAsY37+5XOgU1pyBdZ/nhr7drW2lKSK5Kcfcznv5jknKr6zap6ZpIbThbWWruotbavtbZv58KeAZcJbBKDdk5y297ZkV3DrxiYMp0DjK3b71c6BzanIQc8B1ecXswxRwe11r6T5OFJLk3yC0neOOC2ga1H5wBj0jnA2PQOsCbr/ROtG5OcsZYvqKqzkhxqrb29qq5J8vvr3Daw9egcYEw6Bxib3gE2bF0Dntba9VX14ar6dJL3JnnPKr7sXkneXFVHjxr6l+vZNrD16BxgTDoHGJveAYaw3iN40lo7/5iLLl3xuRevOH3eiuv88Hq3B2xtOgcYk84BxqZ3gI0a8jV4AAAAAJgDAx4AAACAiTPgAQAAAJg4Ax4AAACAiTPgAQAAAJi4db+L1pja4SM58s1vD577g/u/MXhmkvzNHzyiS+79Xn+gS+7CGXsHz7z5UfcZPDNJdr33E11ys7TYJ7eX1vrEHjnSJZdb/crDnt4l9xkf/UKX3Pc9+p5dcmvXrsEzt93troNnJsmRr1zbJRe+T4duv+DsJw2emSTvu/YjXXKfcc8++1BZ6vBzs2r4zKTbz3hYqfbszsIPPmjw3Gf//KMHz0ySAz++rUtu29bneXynP7hs8MyFu9x58MwkWfzmt7rkdqN7T8oRPAAAAAATZ8ADAAAAMHEGPAAAAAATZ8ADAAAAMHEGPAAAAAATZ8ADAAAAMHHdBzxV1ed9NAGOQ+cAY9M7wJh0DnAi3Qc8rbXH9d4GwFE6Bxib3gHGpHOAExnjCJ6bZv/eo6o+UFVXVNWnq+oJvbcNbD06Bxib3gHGpHOAE9k+4rbOT/K+1tprq2pbktNG3Daw9egcYGx6BxiTzgFuY8wBz2VJ3lRVO5L8aWvtipNduar2J9mfJLt1FbB2a+qcRO8AG2ZfBxjT+jtnxx1GWB4wttHeRau19oEkT0zy1SQXV9V/ezvXv6i1tq+1tm9Hdo2yRmDzWGvnzL5G7wDrZl8HGNNGOmfn9tNHWSMwrtEGPFV13yTfbK29Ickbk/zwWNsGth6dA4xN7wBj0jnAscb8E63zkryyqg4nuSnJ7f5vOsAGnBedA4zrvOgdYDznRecAK3Qf8LTW9s7+fUuSt/TeHrC16RxgbHoHGJPOAU5ktD/RAgAAAKAPAx4AAACAiTPgAQAAAJg4Ax4AAACAiTPgAQAAAJg4Ax4AAACAiev+NumDWVqc9wpW7QH/6w1dcpfusKdL7hcuPGfwzDOvaYNnJslpd7lzl9zFb3+7Sy4TVzV4ZDtwcPDMJHnvQ87skvvvrn1/l9yfuvdjB8888pVrB8+EsdSOHdl+93sNnrv0X/rskzzrWS/okrvw0D77D9f88mmDZz7wRdcMnpkkS9870CV3SvvSXXX42Z4+D9u+DhxMPv/lwWNP++YdBs9MktMW+zx+n/QXf98l9y/+8E7Dh3a6DSanTfEJNx5H8AAAAABMnAEPAAAAwMQZ8AAAAABMnAEPAAAAwMQZ8AAAAABMnAEPAAAAwMRtaMBTVWdW1YUrzp9XVe/e+LIAvp/OAcakc4Cx6R1gIzZ6BM+ZSS683WsBDEPnAGPSOcDY9A6wbhsd8Lwuyf2q6oqqev3ssr1V9baq+lxVXVJVlSRV9aiq+suq+uuqel9V3WOD2wa2Hp0DjEnnAGPTO8C6bXTA86okX2itPaK19srZZY9M8rIkD05yTpLHV9WOJL+Z5PmttUcleVOS125w28DWo3OAMekcYGx6B1i37R0yP95auzZJquqKJGcn+cckD03yn2YD521Jvn6ykKran2R/kuzOaR2WCWwSg3TO7Ov1DnB7+nTOtjM6LRfYBIb//apO77hcYF56DHgOrji9ONtGJflMa+2xqw1prV2U5KIkuUPduQ26QmAzGaRzEr0DrEqXzrnjzn+ic4ATGfz3qzsu3EXnwCa00T/RujHJav7L6Zokd62qxyZJVe2oqodscNvA1qNzgDHpHGBsegdYtw0NeFpr1yf5cFV9esWLgB3veoeSPD/J/1VVn0pyRZLHbWTbwNajc4Ax6RxgbHoH2IgN/4lWa+38Yy66dMXnXrzi9BVJnrjR7QFbm84BxqRzgLHpHWC9NvonWgAAAADMmQEPAAAAwMQZ8AAAAABMnAEPAAAAwMQZ8AAAAABMnAEPAAAAwMRt+G3S+X5LX/r7Lrm1a1eX3Hf+yZ8PnvnyH33e4JlJcuTb3+6SC8fV2vCRhw8NntnTzzz7hV1yb3jv4cEz977uDoNnJsnCB67oktvj8cV0tcOHc+Tarw4fXDV8ZpJc+bkusdvOPLNL7vuf9K7BM1906CmDZyZJlhb75LJM9yZJWmtZOnBg+OAemUm3LvvA08/pkpv3Df9r9g2/c+/BM5Pk9Ldd3yWX+XAEDwAAAMDEGfAAAAAATJwBDwAAAMDEGfAAAAAATJwBDwAAAMDEGfAAAAAATNyaBzxV9dKqurqqLqmq51TVq9bwtWdX1flr3SawtekdYEw6BxiTzgGGsn0dX3Nhkqe11q6dnX/nsVeoqu2ttSPH+dqzk5yf5A/WsV1g69I7wJh0DjAmnQMMYk0Dnqr6vSTnJHlvVb0pyXeS7GutvbiqLk5yIMkjk3y4qv4syb+dfWlL8sQkr0tyblVdkeQtrbV/M8y3AWxWegcYk84BxqRzgCGtacDTWvuFqnpmkie31q6rqguOucq9kzyutbZYVe9K8outtQ9X1d4sl9OrkryitfYTQywe2Pz0DjAmnQOMSecAQxr6RZb/fWttcXb6w0l+vapemuTMExxSeEJVtb+qLq+qyw/n4MDLBDYRvQOMSecAY9I5wKoNPeC5+eiJ1trrkvxckj1ZPqTwQWsJaq1d1Frb11rbtyO7Bl4msInoHWBMOgcYk84BVm09L7K8KlV1v9baVUmuqqpHJ3lQkq8kOaPXNoGtTe8AY9I5wJh0DnB7hj6CZ6WXVdWnq+rKJIeTvDfJlUkWq+pTVfXyjtsGtia9A4xJ5wBj0jnASa35CJ7W2tkrTl+c5OLZ6QuOud5LThDxlLVuE9ja9A4wJp0DjEnnAEPpeQQPAAAAACMw4AEAAACYOAMeAAAAgIkz4AEAAACYOAMeAAAAgIkz4AEAAACYuDW/TTq3b9u97tEld/Fr3+yS+0vnPn3wzP/tM+8aPDNJXvOAx3bJbUeOdMmF0VR1iW1Xf7FL7h2fM3zmW7/4F8OHJvnZf/r4LrkwZQsPP7dL7uKV13TJ/cV9zx0880cu/9rgmUny0Ufs7JKb1vrkMl099h0m9jg78o0+v1/VM4Z/Hn/o794zeGaSPPNP93XJ9fvVfDiCBwAAAGDiDHgAAAAAJs6ABwAAAGDiDHgAAAAAJs6ABwAAAGDiDHgAAAAAJu6kA56qOruqPj3Ehqrqy1V11hBZwOakc4Cx6R1gTDoH6MkRPAAAAAATt5oBz/aquqSqrq6qt1XVaUlSVU+tqk9W1VVV9aaq2nWyy4+qqj1V9d6q+vkO3w8wfToHGJveAcakc4AuVjPgeWCS32mtnZvkhiQXVtXuJBcn+enW2sOSbE/yohNdviJrb5J3JfnD1tobBvsugM1E5wBj0zvAmHQO0MVqBjxfaa19eHb695P8WJZL6Uuttc/PLn9Lkiee5PKj/izJm1trb729jVbV/qq6vKouP5yDq1gmsEnMpXMSvQNbmH0dYEw6B+hiNQOedjvn1+LDSZ5ZVXW7G23totbavtbavh3ZdXtXBzaPuXROondgC7OvA4xJ5wBdrGbAc5+qeuzs9PlJPpTkmiRnV9X9Z5f/bJK/PMnlR70myXeS/PZGFw5sWjoHGJveAcakc4AuVjPguSbJL1bV1UnulOR3W2sHkrwwyb+vqquSLCX5vRNdfkzeLyXZU1W/NtQ3AWwqOgcYm94BxqRzgC62n+yTrbUvJ3nQCT73/iSPXMPlZ684+8K1LBLYGnQOMDa9A4xJ5wA9reYIHgAAAABOYQY8AAAAABNnwAMAAAAwcQY8AAAAABNnwAMAAAAwcSd9F61TysK24TOXFofPTHLk777SJbeXdvjQ4Jn/yzmPGTwzSd731cu75D7jXt/3xgTDaK1PblWX2IU9e7rk5uY+sT3V9m3ZduadB89tBw4Onrkc3OextnDnO3XJXfzGNwfP/BcPeOrgmUmy8NCz++Te9N0uuV9/1r265N7t8pu65C787bWDZ9Z/6bDP0Fnt2J7td7374LlHvv6NwTOTZOnKa7rkbrvL8L2bJN96zv0Hz7zs+WcMnpkkB378rC65p/3l1V1ye/1cq3PP6ZKbz3958Mg60Ge/rKfasSPb737PwXOPfPVrg2cmSarPcQkLe0/rkrt00/A/M5/1rBcMnpkkX7pkR5fc+7/sW11y2/e+1yV36aZOv7Bs67RPcuD4FzuCBwAAAGDiDHgAAAAAJs6ABwAAAGDiDHgAAAAAJs6ABwAAAGDiDHgAAAAAJs6ABwAAAGDi1jTgqaozq+rCFefPq6p3rzHjgqq651q+BtiadA4wNr0DjEnnAENa6xE8Zya58HavdXIXJFFAwGroHGBsegcYk84BBrPWAc/rktyvqq6oqtfPLttbVW+rqs9V1SVVVUlSVa+pqsuq6tNVdVEte36SfUkumWXsGfB7ATYfnQOMTe8AY9I5wGDWOuB5VZIvtNYe0Vp75eyyRyZ5WZIHJzknyeNnl/9Wa+3RrbWHJtmT5Cdaa29LcnmS/2aW8b0Tbaiq9lfV5VV1+eEcXOMygU1itM5Jbts7h5YOdPmGgFPeXPZ1Di2dtJ6AzUvnAIMZ4kWWP95au7a1tpTkiiRnzy5/clV9rKquSvKUJA9ZS2hr7aLW2r7W2r4d2TXAMoFNokvnJLftnZ0Lu4dbMTB13fd1di74T3fgFjoHWJftA2SsPLxmMcn2qtqd5HeS7GutfaWq/lUSvy0BQ9A5wNj0DjAmnQOsy1qP4LkxyRmruN7RsrmuqvYmef46MgB0DjA2vQOMSecAg1nTgKe1dn2SD89e2Ov1J7nePyZ5Q5JPJ3lfkstWfPriJL/nRcCA26NzgLHpHWBMOgcY0pr/RKu1dv4xF1264nMvXnH61UlefZyvf3uSt691u8DWpHOAsekdYEw6BxjKEC+yDAAAAMAcGfAAAAAATJwBDwAAAMDEGfAAAAAATJwBDwAAAMDErfldtOahqrKwc8fguUsHFgfPnKLa3uFhsG3b8JlJnrR/f5fcHecd6ZJ7w312dck967Lru+S2HX3ut1zRJ7andmQxi9f/w7yXMXdL3/3uvJewau1In+dxPnNNl9jvPXtfl9zT/tk3u+Rue+eBLrlHvvOdwTNbm+DP99bSDh+e9ypWry11iV389re75N7tr+40eGb7Rp+17r3x5i65Of20Prln3blL7N/+9PD3WZLc7y0dnmdf3jl8Zmft8OEc+erXOgS34TOTpFOvL914Y5fcHpY+dXWX3HNe2KcbPvv/PKRL7lkf7TPCOOvy4fdHkmTpqj77kSfiCB4AAACAiTPgAQAAAJg4Ax4AAACAiTPgAQAAAJg4Ax4AAACAiTPgAQAAAJg4Ax4AAACAiZvLgKeqPjKP7QJbl94BxqRzgDHpHCCZ04Cntfa4eWwX2Lr0DjAmnQOMSecAyfyO4LlpHtsFti69A4xJ5wBj0jlAkmyf9wJOpKr2J9mfJLvr9DmvBtgKbtM7OW3OqwE2u9t0zsLeOa8G2Ozs58Dmd8q+yHJr7aLW2r7W2r6d2TXv5QBbwMre2aF3gM5us6+zsHveywE2Ofs5sPmdsgMeAAAAAFbHgAcAAABg4gx4AAAAACZuXm+T7pUEgVHpHWBMOgcYk84BEkfwAAAAAEyeAQ8AAADAxBnwAAAAAEycAQ8AAADAxBnwAAAAAEzc9nkvYDVaa1k6eHDey9i02pEjw4f2yEyy+90f75Jb2/s8Ff7ki5d2yd3/yOd0yc2hw31yYcpa6xK76z2Xdcm99A1XdMl9xtce2SWXZe3IYhavu37ey1i9Ts+LXhY/+/l5L2HVlm68sUtur32d3/7CpV1yX7LvuV1yF6+7bvDMtjS931OqKgu7dg2eu3TgwOCZ9LV0oM/j99xfG/65liQ3PPxuXXKv+R9P65J77q/0WW++fvyLHcEDAAAAMHEGPAAAAAATZ8ADAAAAMHEGPAAAAAATZ8ADAAAAMHEGPAAAAAATZ8ADAAAAMHGDD3iq6l9V1Stmp3+1qp52nOucV1XvHnrbwNajc4Ax6RxgTDoHWIvtPcNba6/pmQ+wks4BxqRzgDHpHOD2DHIET1X9z1X1+ar6UJIHrrj84qp6/uz0M6vqc1X1iST/fIjtAluTzgHGpHOAMekcYL02POCpqkcl+Zkkj0jy7CSPPs51did5Q5L/Osmjktx9Fbn7q+ryqrr8cA5udJnAJtGrc2Zfp3eA29A5wJjG6pxDOgc2pSGO4HlCkne01r7bWrshyTuPc50HJflSa+1vWmstye/fXmhr7aLW2r7W2r4d2TXAMoFNokvnJHoHOC6dA4xplM7ZqXNgU/IuWgAAAAATN8SA5wNJfrKq9lTVGVk+VPBYn0tydlXdb3b+BQNsF9iadA4wJp0DjEnnAOu24XfRaq19oqr+OMmnknwryWXHuc6Bqtqf5D1V9d0kH0xyxka3DWw9OgcYk84BxqRzgI0Y5G3SW2uvTfLa41x+wYrT/zHLfy8KsCE6BxiTzgHGpHOA9fIaPAAAAAATZ8ADAAAAMHEGPAAAAAATZ8ADAAAAMHEGPAAAAAATN8i7aI2iOsyi2uLwmXRV2/s8ZBceeL8uuf/9Tz24S+6hR+/qkvvl51SX3LyoTywjqE6Pidb65HbQq3eWHvOQLrk/8K5Hd8k958lLXXJ3feYrg2fWddPZvbmNHs+3CT3XmFnYNu8VrMnz/u9f7pK794l99tNvvM8DBs88fMlfDZ7ZXVWy4P/6SRZ29/m9on3zui65qbt1ib3PH/Xp3r99yTldcvM/Hf9iz2oAAACAiTPgAQAAAJg4Ax4AAACAiTPgAQAAAJg4Ax4AAACAiTPgAQAAAJi4uQx4quojs3/Prqrz57EGYOvQOcCYdA4wJp0DHDWXAU9r7XGzk2cnUUJAVzoHGJPOAcakc4Cj5nUEz02zk69L8oSquqKqXj6PtQCbn84BxqRzgDHpHOCo7XPe/quSvKK19hNzXgewNegcYEw6BxiTzoEtbt4DnhOqqv1J9ifJ7pw259UAW4HeAcakc4Ax3aZz6vQ5rwbo4ZR9F63W2kWttX2ttX07smveywG2AL0DjEnnAGNa2Tk7a/e8lwN0MO8Bz41JzpjzGoCtQ+cAY9I5wJh0Dmxx8x7wXJlksao+5YXAgBHoHGBMOgcYk86BLW4ur8HTWts7+/dwkqfMYw3A1qFzgDHpHGBMOgc4at5H8AAAAACwQQY8AAAAABNnwAMAAAAwcQY8AAAAABNnwAMAAAAwcQY8AAAAABM3l7dJX6vavi3b7nTm4LmL110/eCbTtHj133bJXdi9q0vu0lMf2iX3IQ/+Spfcv++S2lft2pnt9z578NzFr3598MwkaQcPdslNa11ia8fOwTPb4UODZyZJ7dnTJXf756/tknvOH5/dJffme+zoknvgrHMGz1z88z7d21MtLGRh797Bc5duvHHwTPqqHX12z6uqS+6e65a65B64c5//h95x4/A/12px8Mj+Kqntk/hVkKM6PYfbYp8H8MIZw/9MS5I7Xva1Lrk3PeweXXIP37vTPvoJOIIHAAAAYOIMeAAAAAAmzoAHAAAAYOIMeAAAAAAmzoAHAAAAYOIMeAAAAAAmzoAHAAAAYOIMeAAAAAAmzoAHAAAAYOIMeAAAAAAmbvu8F3AiVbU/yf4k2b2wd86rAbaC2/TO9jPmvBpgs7tN59Tpc14NsNnpHNj8TtkjeFprF7XW9rXW9u1c2D3v5QBbwG16Z9tp814OsMndpnPKvg7Ql9+vYPM7ZQc8AAAAAKzO3Ac8VfXGqto373UAW4POAcamd4Ax6RzYuub+GjyttZ+b9xqArUPnAGPTO8CYdA5sXXM/ggcAAACAjTHgAQAAAJg4Ax4AAACAiTPgAQAAAJg4Ax4AAACAiTPgAQAAAJi4aq3New23q6q+neTvVnn1s5Jc12EZcqeXO6W1bvbc+7bW7tphDd2cAr1zKtxvcjd37pTWutZcnbM+U8qd0lrlTjNX59xqSveb3H6Zck+t3OP2ziQGPGtRVZe31vbJlTultcqdNo9fuVPMndJae+ZO0dRuY49fuVPM1Tm3mtL9Jrdfptxp5PoTLQAAAICJM+ABAAAAmLjNOOC5SO64uVV10zHnL6iq39po7izr0qr6vsPUquriqvpSVV0x+3jEajMHILdv7hT1uC2mdr9t9s6pqnptVX2+qq6uqpeuJXcAHmM6Z6Wp3caTfvzOqXM+uGIf52tV9adryR2AXJ2z0pTut02RO0DvnHCtJ+mdp1bVJ2a986Gquv9acjdI7kC5m+41eBhfVd3UWtu74vwFSfa11l48QPalSV7RWrv8mMsvTvLu1trbNroNYFrm1DkvTPLkJBe01paq6m6ttW9tdHvAqW8enXPMdd6e5M9aa2/d6PaAaZjTvs7nk/yz1trVVXVhkse01i7Y6PYY12Y8godTSFXdtareXlWXzT4eP7v8MVX1V1X1yar6SFU9cHb5nqr6o9n/kL8jyZ65fgPApHTsnBcl+dXW2lKSGO4ASf/9nKq6Q5KnJDnRETzAFtOxd1qSO8xO3zHJ17p/Mwxu+7wXwKawp6quWHH+zkneOTv9b5P8m9bah6rqPknel+TcJJ9L8oTW2pGqelqS/zPJ87L8S9R3W2ur45WBAAACZklEQVTnVtUPJfnESbb72qp6TZL3J3lVa+3gsN8WcIqaR+fcL8lPV9Vzk3w7yUtba38z+HcGnIrmtZ+TJD+Z5P2ttRsG/H6AU988eufnkvyHqvpekhuS/Ojg3xXdGfAwhO+11m55DZyjhxDOzj4tyYOr6uin71BVe7M8FX5LVT0gy9PiHbPPPzHJbyRJa+3KqrryBNv8l0m+kWRnlv9W8VeS/OpQ3xBwSptH5+xKcqC1tq+q/nmSNyV5wnDfEnAKm0fnHPWCJG8c4psAJmUevfPyJM9urX2sql6Z5NezPPRhQgx46G0hyY+21g6svHD2ImF/0Vp7blWdneTStYS21r4+O3mwqt6c5BUbXyqwCXTpnCTXJvmT2el3JHnzxpYJbBK9OidVdVaSxyR57saXCWwig/dOVd01ycNbax+bXfTHSf7jIKtlVF6Dh97+PMlLjp6pW9/t6o5Jvjo7fcGK638gyfmz6z40yQ8dL7Sq7jH7t7J8+PKnh1w0MFldOifLr3/x5NnpJyX5/DDLBSauV+ckyfOz/IYSB05yHWDr6dE730lyx6r6wdn5pye5erglMxYDHnp7aZJ9VXVlVX02yS/MLv+1JP+6qj6Z2x5J9rtJ9lbV1Vn+k6u/PkHuJVV1VZKrkpyV5P/osnpganp1zuuSPG/WO/86DlkGlvXqnCT5mSR/2GHNwLQN3juttSNJfj7J26vqU0l+NskrO34PdOJt0gEAAAAmzhE8AAAAABNnwAMAAAAwcQY8AAAAABNnwAMAAAAwcQY8AAAAABNnwAMAAAAwcQY8AAAAABNnwAMAAAAwcf8/CUI/9XIP7fUAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 1152x576 with 8 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Real translation: this is the first book i've ever done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WO2mA6lAO5rR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}