HW4_DNN.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "HW4_DNN.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "cH9AxD0t1UWj",
        "JJ6gPUEzGVw1",
        "mm0HmkJyZDc7",
        "eqJa8-p8oruu",
        "qM1pU5x37fyR",
        "Y8W2Lo4NWaSn",
        "pneH1f0dHOau",
        "hXgiAe3LM8zo",
        "pKb1iEhLgB8j"
      ]
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "79be990fb4d94d4eb00e7e40c34a7dc0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d6bebec478ff445596d2d20e3e41d8d8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7357c2dbe01b465a8025bb04079e1ba6",
              "IPY_MODEL_789af892e98d4cd9945927f18d344b33"
            ]
          }
        },
        "d6bebec478ff445596d2d20e3e41d8d8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7357c2dbe01b465a8025bb04079e1ba6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5c87f37dea28475989e457a281b49bcb",
            "_dom_classes": [],
            "description": " 96%",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 5000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 4813,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_81ef136b29f448bbac4feb43dfe71d3b"
          }
        },
        "789af892e98d4cd9945927f18d344b33": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_dec40c6f7fe943f28b70dcf421feeeaf",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 4813/5000 [56:17&lt;01:24,  2.21it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1ec6627be3104bd1ac8f7527b49c612c"
          }
        },
        "5c87f37dea28475989e457a281b49bcb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "81ef136b29f448bbac4feb43dfe71d3b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dec40c6f7fe943f28b70dcf421feeeaf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1ec6627be3104bd1ac8f7527b49c612c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "103187e7a8c54bf287f1c280d87b24ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_13862922e0b7405aa69aacb30866f962",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5c53dcfed97c4acc8bb0c68684c358db",
              "IPY_MODEL_3c046fa78c6d4517aced2b9eac42a1b0"
            ]
          }
        },
        "13862922e0b7405aa69aacb30866f962": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5c53dcfed97c4acc8bb0c68684c358db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1df95b56f524432c8de6767237c32a36",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 535,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 535,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_35b71d1b0a124dbaad77f6a66dcf337a"
          }
        },
        "3c046fa78c6d4517aced2b9eac42a1b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4008d352b6e84b908bb9e59fc7b07846",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 535/535 [20:31&lt;00:00,  2.30s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9e06d9aa78374fb5b8e435a7926cd1fc"
          }
        },
        "1df95b56f524432c8de6767237c32a36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "35b71d1b0a124dbaad77f6a66dcf337a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4008d352b6e84b908bb9e59fc7b07846": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9e06d9aa78374fb5b8e435a7926cd1fc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "08c8a169418142a28e90cc2cee731fd0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_37653c2234204b43afd44f9e409e18c3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4076ba1f2023409c9f13da22659d4456",
              "IPY_MODEL_96880343dafb49678199fe83d5dc4909"
            ]
          }
        },
        "37653c2234204b43afd44f9e409e18c3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4076ba1f2023409c9f13da22659d4456": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_595e5459dbd64350a1b1927da3895f99",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 535,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 535,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fce7fe73f9614d85a4cc788fe850de3d"
          }
        },
        "96880343dafb49678199fe83d5dc4909": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0dbeb0cf42604b9284961c661d7bc440",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 535/535 [14:33&lt;00:00,  1.63s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_73f2e6e91a1f461493a30bddb4dca257"
          }
        },
        "595e5459dbd64350a1b1927da3895f99": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fce7fe73f9614d85a4cc788fe850de3d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0dbeb0cf42604b9284961c661d7bc440": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "73f2e6e91a1f461493a30bddb4dca257": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4b70b01b2f784ab7b4d03dc0690a8133": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_11f7a56252c34ac9a610efa568f449c3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c8432bf3291b41658bdea268bbe4a52c",
              "IPY_MODEL_03e6704894e4460cbf9ac756b1b7fbf5"
            ]
          }
        },
        "11f7a56252c34ac9a610efa568f449c3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c8432bf3291b41658bdea268bbe4a52c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_553feaf967734e659b3e40e864890591",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 535,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 535,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9724c16f39804e65a2339f4eb0888d87"
          }
        },
        "03e6704894e4460cbf9ac756b1b7fbf5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_35d8838d189b43b6a89e56456ce5cda6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 535/535 [08:38&lt;00:00,  1.03it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_aec6f668a23f4ef3ab62edc211ee50e1"
          }
        },
        "553feaf967734e659b3e40e864890591": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9724c16f39804e65a2339f4eb0888d87": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "35d8838d189b43b6a89e56456ce5cda6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "aec6f668a23f4ef3ab62edc211ee50e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a81de214bedf4591962bf94e5fbe6d05": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_11ed2eec12394dcebbd6675ec09eae68",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_44b106d095c249a182e4be352474017e",
              "IPY_MODEL_ef737862bdc04cab8e983aae46c8d61a"
            ]
          }
        },
        "11ed2eec12394dcebbd6675ec09eae68": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "44b106d095c249a182e4be352474017e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_564e39703608415f877eb6ae697de7a0",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 535,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 535,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4a389d42259f4a0b91fef8be9bb3331b"
          }
        },
        "ef737862bdc04cab8e983aae46c8d61a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5ad8627d97bc484a8ad7e597bba19059",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 535/535 [01:28&lt;00:00,  6.02it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_96fd40b4f0f947cd86f8239cb0c05dfd"
          }
        },
        "564e39703608415f877eb6ae697de7a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4a389d42259f4a0b91fef8be9bb3331b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5ad8627d97bc484a8ad7e597bba19059": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "96fd40b4f0f947cd86f8239cb0c05dfd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3082e9e6a6ed41f0915fa2e83a517855": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_216cc2fc3b52449081bd009710f6610a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d8e3054ea7814cd49bc2c853ba00b438",
              "IPY_MODEL_7378bead921449b197d2d07a69c9be89"
            ]
          }
        },
        "216cc2fc3b52449081bd009710f6610a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d8e3054ea7814cd49bc2c853ba00b438": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7fb8d548288c451981066b81db43907c",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 535,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 535,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_776d8e3df4404e55b1940d4484be56a1"
          }
        },
        "7378bead921449b197d2d07a69c9be89": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9092ec82a52c4d7fbe5b23ea5b49929e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 535/535 [23:00&lt;00:00,  2.58s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e6608a381424445382708ea0f0ed2f30"
          }
        },
        "7fb8d548288c451981066b81db43907c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "776d8e3df4404e55b1940d4484be56a1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9092ec82a52c4d7fbe5b23ea5b49929e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e6608a381424445382708ea0f0ed2f30": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3b5227a489704d3c900eeb3a462bd282": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_912aaef2f7d347d680a434c28f5094f9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_983239dac92b44a3bf23bef0a3b043f9",
              "IPY_MODEL_ce572565bc8340589c4dccfc59499389"
            ]
          }
        },
        "912aaef2f7d347d680a434c28f5094f9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "983239dac92b44a3bf23bef0a3b043f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e5ee78621e0743719c1f1affa1ef9342",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 535,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 535,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e2ab5b323e2f40779f69d3cbc9f48183"
          }
        },
        "ce572565bc8340589c4dccfc59499389": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_01bd995c470b40b0b8044f1522b72ca3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 535/535 [01:26&lt;00:00,  6.21it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3f56b2be43c74873bd9e1f0fb7f0d5f1"
          }
        },
        "e5ee78621e0743719c1f1affa1ef9342": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e2ab5b323e2f40779f69d3cbc9f48183": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "01bd995c470b40b0b8044f1522b72ca3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3f56b2be43c74873bd9e1f0fb7f0d5f1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6bd9fbd4d1584092807987bf75cbd6b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_50c88e7de0ac4237a27388f0e048d874",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8174f6f32337401f820325273ae84a5e",
              "IPY_MODEL_0c87e696b3f54a1fad11004c923e9fd8"
            ]
          }
        },
        "50c88e7de0ac4237a27388f0e048d874": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8174f6f32337401f820325273ae84a5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_53dc7fbe0da84bc6b988e05de9bc7ed1",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 535,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 535,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_799a086984254677a0f75b888ac3e6d8"
          }
        },
        "0c87e696b3f54a1fad11004c923e9fd8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_20678248d9444021b140a2bab9ad6d06",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 535/535 [11:09&lt;00:00,  1.25s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ca8e9854ae3e49b5a66ecd87e04de47e"
          }
        },
        "53dc7fbe0da84bc6b988e05de9bc7ed1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "799a086984254677a0f75b888ac3e6d8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "20678248d9444021b140a2bab9ad6d06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ca8e9854ae3e49b5a66ecd87e04de47e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "adcaf598152841c89569fdb0831f46c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_fb0d56e5b3484062bb2d7956750933ef",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_52200d85df18452a912040057168d8b1",
              "IPY_MODEL_831e5d06718c457abbd10ce957debeb5"
            ]
          }
        },
        "fb0d56e5b3484062bb2d7956750933ef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "52200d85df18452a912040057168d8b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_270c66b7748e411e86beab3b320af455",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 535,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 535,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b57d3a2256da42e9a82b7cb82245a307"
          }
        },
        "831e5d06718c457abbd10ce957debeb5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1e9a54e5fca646c9a7945eac9838581c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 535/535 [05:15&lt;00:00,  1.70it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_60172a871b024528a3ed118626522b8c"
          }
        },
        "270c66b7748e411e86beab3b320af455": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b57d3a2256da42e9a82b7cb82245a307": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1e9a54e5fca646c9a7945eac9838581c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "60172a871b024528a3ed118626522b8c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a32d997d0cec451ab443f0d1e2dacb5c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d549d1053f6b4cd3a9c98fe80208fccb",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_332395b1ef6641f5bcbfa133f88b1375",
              "IPY_MODEL_64599e3a0b80469fbf5bf0fb97655da4"
            ]
          }
        },
        "d549d1053f6b4cd3a9c98fe80208fccb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "332395b1ef6641f5bcbfa133f88b1375": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2a5f82b38d4348ac9dfc8a33d2beb500",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 535,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 535,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7040547c40804dcd9020166408347054"
          }
        },
        "64599e3a0b80469fbf5bf0fb97655da4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_cbac7175a6384253b163299ad469966e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 535/535 [01:27&lt;00:00,  6.13it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_31befd519b2744dc9139d685090bb453"
          }
        },
        "2a5f82b38d4348ac9dfc8a33d2beb500": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7040547c40804dcd9020166408347054": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cbac7175a6384253b163299ad469966e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "31befd519b2744dc9139d685090bb453": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1d229d2b4ae147f0a07842bd01eb68ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_20b7c7394f8b4cefa1439d0c12cd649f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_88bb4390fdbc49258913f6da7ad74205",
              "IPY_MODEL_766075418f0f49049d1c7aeafb4a4b25"
            ]
          }
        },
        "20b7c7394f8b4cefa1439d0c12cd649f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "88bb4390fdbc49258913f6da7ad74205": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_92e8c6e1fbcf4c9296c373107d4eaac4",
            "_dom_classes": [],
            "description": " 96%",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 5000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 4813,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d3544e728d73447d88ff4ef0caff2bd4"
          }
        },
        "766075418f0f49049d1c7aeafb4a4b25": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b70ed59302a34f2bb991aebbc7fd0ddf",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 4813/5000 [57:15&lt;01:28,  2.12it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e54ed8ec9f56400c8f19bbdf4d038a3d"
          }
        },
        "92e8c6e1fbcf4c9296c373107d4eaac4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d3544e728d73447d88ff4ef0caff2bd4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b70ed59302a34f2bb991aebbc7fd0ddf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e54ed8ec9f56400c8f19bbdf4d038a3d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0605a2f5fd8c4b62b69edb3fb23015df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f20845546eda4539bfe972dd55f18717",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6323cdf06f96463dae17aaa9c4eb9ec9",
              "IPY_MODEL_8d8f0ec30bc3443cbc685259f2c83629"
            ]
          }
        },
        "f20845546eda4539bfe972dd55f18717": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6323cdf06f96463dae17aaa9c4eb9ec9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_66eadeb29e2e48b8aa237448e2703512",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 535,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 535,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e561633365a84a5dbd83b1e8b41bb793"
          }
        },
        "8d8f0ec30bc3443cbc685259f2c83629": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_df863cbfda3040d7a7867ab96a3ef478",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 535/535 [16:52&lt;00:00,  1.89s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_059cc0a36426450381df6122f294b7c9"
          }
        },
        "66eadeb29e2e48b8aa237448e2703512": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e561633365a84a5dbd83b1e8b41bb793": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "df863cbfda3040d7a7867ab96a3ef478": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "059cc0a36426450381df6122f294b7c9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f7a6ca09cb2b4c7797ba0f98430a6866": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6409278f894a47ecb484e0d1fb94fdce",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_acf4771e6efd41a68e8a2c42872cca8d",
              "IPY_MODEL_2f61fad46b174b28b945520232a6bf6b"
            ]
          }
        },
        "6409278f894a47ecb484e0d1fb94fdce": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "acf4771e6efd41a68e8a2c42872cca8d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9cff60fce822429eb1a52dea6f6aec32",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 535,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 535,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9df6adfb8899408483b095c835e58955"
          }
        },
        "2f61fad46b174b28b945520232a6bf6b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b4fa9dd936a647a496d0c94e594cc70a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 535/535 [10:48&lt;00:00,  1.21s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6a2b8bdb81e742df9ad55a91146eb4ef"
          }
        },
        "9cff60fce822429eb1a52dea6f6aec32": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9df6adfb8899408483b095c835e58955": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b4fa9dd936a647a496d0c94e594cc70a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6a2b8bdb81e742df9ad55a91146eb4ef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "aa026421293549afa11196fa0d06e2a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3e87d692436a47fd898215c8f82ba3bd",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8bdb740d9ba64b778854c4bdb15bddfc",
              "IPY_MODEL_4ee532e0a6224b92a26852dd47f1e2de"
            ]
          }
        },
        "3e87d692436a47fd898215c8f82ba3bd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8bdb740d9ba64b778854c4bdb15bddfc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_445276748bf44d12807f5a29afe14c85",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 535,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 535,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b0c3a7176d164f0c913311981a28785c"
          }
        },
        "4ee532e0a6224b92a26852dd47f1e2de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d71508d2848d4733bca3cb3d2dd821f9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 535/535 [04:47&lt;00:00,  1.86it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_948f29f553bd4b44bd5b0f82076b281d"
          }
        },
        "445276748bf44d12807f5a29afe14c85": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b0c3a7176d164f0c913311981a28785c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d71508d2848d4733bca3cb3d2dd821f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "948f29f553bd4b44bd5b0f82076b281d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d34e99f0ea4144e6a7a94c030030c59f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ee3c9df8970946088c31359fd5ded64e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_68f0dde2f22d4fe7bc65cf0be8f18bd2",
              "IPY_MODEL_61006b21dd264cf2b6ef90430724ae23"
            ]
          }
        },
        "ee3c9df8970946088c31359fd5ded64e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "68f0dde2f22d4fe7bc65cf0be8f18bd2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_68634acf1af4485c932a1ddaabb92467",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 535,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 535,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a758531ed2584c03966021adf0ad6261"
          }
        },
        "61006b21dd264cf2b6ef90430724ae23": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_82c510a02db4432696e36a13de338b03",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 535/535 [26:55&lt;00:00,  3.02s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f0ce220e9b8d4329811771e11d1b4dac"
          }
        },
        "68634acf1af4485c932a1ddaabb92467": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a758531ed2584c03966021adf0ad6261": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "82c510a02db4432696e36a13de338b03": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f0ce220e9b8d4329811771e11d1b4dac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "16b1ae35ffd74bc98ef4bc7523524aed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c8045c741b01455aaa9371b3ba07e50f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_22afe2b68d65415fb46c827c7dd97809",
              "IPY_MODEL_87e337067d814094967507acd350be80"
            ]
          }
        },
        "c8045c741b01455aaa9371b3ba07e50f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "22afe2b68d65415fb46c827c7dd97809": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a6fe6eaf5474437486632b51388d3755",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 535,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 535,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a57d7d3a92b848058776dce01c6c6e8e"
          }
        },
        "87e337067d814094967507acd350be80": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_22048c9838c74bc593da208e812edef0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 535/535 [20:51&lt;00:00,  2.34s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_66831f3c2b9440c98107c162066603d5"
          }
        },
        "a6fe6eaf5474437486632b51388d3755": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a57d7d3a92b848058776dce01c6c6e8e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "22048c9838c74bc593da208e812edef0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "66831f3c2b9440c98107c162066603d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "afc4b3f7af6f48238794450d90662082": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_08b26a14bdf54ea0899e236eb15a9699",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7366d75e65c04958a9d6195b14042291",
              "IPY_MODEL_e0071200473d49058ba25589a9aed2c0"
            ]
          }
        },
        "08b26a14bdf54ea0899e236eb15a9699": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7366d75e65c04958a9d6195b14042291": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_43939e07e3144b97b9a23e5b5a7396e6",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 535,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 535,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dd4e16a8091b437d90f93e2d85526f69"
          }
        },
        "e0071200473d49058ba25589a9aed2c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e21c1e1230764cbca572a7cec1f9ce2c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 535/535 [01:29&lt;00:00,  5.96it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_530c2e0b7dab4e42ab5646f9e6b88f03"
          }
        },
        "43939e07e3144b97b9a23e5b5a7396e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dd4e16a8091b437d90f93e2d85526f69": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e21c1e1230764cbca572a7cec1f9ce2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "530c2e0b7dab4e42ab5646f9e6b88f03": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1f5853e0803d42298a4481dbee217c7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_cd827ecc14be43acb2201773824931e8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_997eb1ceb61a4278846212428f890ad5",
              "IPY_MODEL_636a9792555a40438c5e367ad20cdddf"
            ]
          }
        },
        "cd827ecc14be43acb2201773824931e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "997eb1ceb61a4278846212428f890ad5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cb81f1cb82484d2a85aeaef84498adf4",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 535,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 535,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9c96446d5e6944ba8897cd734b4ca20c"
          }
        },
        "636a9792555a40438c5e367ad20cdddf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fde49670538348908feae72d735010dd",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 535/535 [08:50&lt;00:00,  1.01it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_92d39007791145f8aecfb3a4a556a196"
          }
        },
        "cb81f1cb82484d2a85aeaef84498adf4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9c96446d5e6944ba8897cd734b4ca20c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fde49670538348908feae72d735010dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "92d39007791145f8aecfb3a4a556a196": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c350b3cc6be44435ab9513ffbfdd534f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_fa28262d042f44448a657e4b85b4a4f8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_01032022e4ae4589a4a922ed3c56ae53",
              "IPY_MODEL_06fe92a7afd54d9f8354d94defc2f355"
            ]
          }
        },
        "fa28262d042f44448a657e4b85b4a4f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "01032022e4ae4589a4a922ed3c56ae53": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_17e5e2f0764343588b4b05a57812aed1",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 535,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 535,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6f4b9aece3054fa7bae7a922d506aca2"
          }
        },
        "06fe92a7afd54d9f8354d94defc2f355": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_afc7292e11784eda93c173d7300b1992",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 535/535 [01:26&lt;00:00,  6.15it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b2b9dd12d9484553a52aa5e25be363a0"
          }
        },
        "17e5e2f0764343588b4b05a57812aed1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6f4b9aece3054fa7bae7a922d506aca2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "afc7292e11784eda93c173d7300b1992": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b2b9dd12d9484553a52aa5e25be363a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "856c7f0665dd4b1a8e20c63287455482": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d8392e8dba6d4518964cb208da74e97d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_cfb807635c2e410dade5ac5b91288bda",
              "IPY_MODEL_f58c295313c94346b663706bc2ad23a4"
            ]
          }
        },
        "d8392e8dba6d4518964cb208da74e97d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cfb807635c2e410dade5ac5b91288bda": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_dd916a24ebfe4358b8f3b31dfe339af9",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 535,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 535,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3cb99ab7eb0a44d8b5da66eb5bd2e993"
          }
        },
        "f58c295313c94346b663706bc2ad23a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_37659054e0c94312900d986de97f1360",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 535/535 [01:25&lt;00:00,  6.25it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e147b70b74e740bdaf7e3234df35368b"
          }
        },
        "dd916a24ebfe4358b8f3b31dfe339af9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3cb99ab7eb0a44d8b5da66eb5bd2e993": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "37659054e0c94312900d986de97f1360": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e147b70b74e740bdaf7e3234df35368b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "09e08c49f9544dd094c8c66856bc16ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1d938503ada349d49db03e07076aa4e5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0eb930bff3464e078ba75074c5785f74",
              "IPY_MODEL_71b30636499f406c8e462e6eb188b999"
            ]
          }
        },
        "1d938503ada349d49db03e07076aa4e5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0eb930bff3464e078ba75074c5785f74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2c798f31332c4b759b2fa7febfce93bd",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 250,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 250,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_79c02617407b42d1864f66eddc3a25f2"
          }
        },
        "71b30636499f406c8e462e6eb188b999": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e35c15bdd93e40ef85fa396608bceaed",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 250/250 [02:46&lt;00:00,  1.50it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3b17e7fbcb104130a86a2a7d074c124c"
          }
        },
        "2c798f31332c4b759b2fa7febfce93bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "79c02617407b42d1864f66eddc3a25f2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e35c15bdd93e40ef85fa396608bceaed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3b17e7fbcb104130a86a2a7d074c124c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a2808f06173f427eb9d8be18570b68a8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0894d23cd20946e797cb1757aa1a529c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7064339856d040f68e789713d52b8247",
              "IPY_MODEL_a7f3dceb2ece4ca8982b4a4d1a50a426"
            ]
          }
        },
        "0894d23cd20946e797cb1757aa1a529c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7064339856d040f68e789713d52b8247": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_96b8571b6e3e4b7190ebc2665a3d1101",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 10,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 10,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b106ad01751d413a9bc7f64410cfa383"
          }
        },
        "a7f3dceb2ece4ca8982b4a4d1a50a426": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c7157f5c93f24b399c2546952ffe985b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 10/10 [00:00&lt;00:00, 19.12it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5fb185d7b19941aaae13526991e3aa30"
          }
        },
        "96b8571b6e3e4b7190ebc2665a3d1101": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b106ad01751d413a9bc7f64410cfa383": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c7157f5c93f24b399c2546952ffe985b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5fb185d7b19941aaae13526991e3aa30": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kTtc369Zw8_n"
      },
      "source": [
        "### Problem Definition  \n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ozDKkPgVxA6z"
      },
      "source": [
        "In this assignment we implemented a Transformer model from the Attention is All You Need paper[1]. A Sequence-to-Sequence model can be used for machine translation, speech recognition, and many other purposes.  The Transformer does not use any recurrence, like the Convolutional Sequence-to-Sequence model. It also does not use any convolutional layers. The model is instead entirely composed of linear layers, attention mechanisms, and normalization.\n",
        "\n",
        "\n",
        "In this assignment, we used the Transformer Network to translate English sentences into Persian and evaluated the translation by the BLEU and NIST scores. We used the AFEC Dataset, which was prepared by the NLP lab at Tehran University. \n",
        "."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ivDsSt38jZ_U",
        "outputId": "4ad1ba03-c181-437c-fe4e-de6c219ca3a4"
      },
      "source": [
        "pip install pyonmttok"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pyonmttok\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/90/73/034c3e0584322e3f3f03c0965c8a83df0ab7ae5ca65172203cd606ffe8ce/pyonmttok-1.26.4-cp37-cp37m-manylinux1_x86_64.whl (14.3MB)\n",
            "\u001b[K     |████████████████████████████████| 14.3MB 218kB/s \n",
            "\u001b[?25hInstalling collected packages: pyonmttok\n",
            "Successfully installed pyonmttok-1.26.4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aGP7yieVvXuR",
        "outputId": "ca2ad8ad-e004-433c-909d-0588f9882284"
      },
      "source": [
        "#Importing required libraries \n",
        "import numpy as np\n",
        "import math \n",
        "import os, sys \n",
        "\n",
        "import random \n",
        "import matplotlib.pyplot as plt\n",
        "import string\n",
        "import nltk \n",
        "nltk.download('punkt')\n",
        "\n",
        "import pyonmttok\n",
        "import torch \n",
        "from torch import Tensor\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "# import torchvision.transforms as transforms\n",
        "from typing import Callable, List, Optional, Tuple\n",
        "from torch.nn.utils.rnn import pad_sequence \n",
        "from torch.optim.lr_scheduler import MultiStepLR\n",
        "import torch.optim as optim\n",
        "from torch.nn import TransformerDecoder, TransformerDecoderLayer\n",
        "\n",
        "from tqdm.notebook import tqdm\n",
        "from nltk.translate.bleu_score import corpus_bleu,sentence_bleu\n",
        "from nltk.translate.nist_score import corpus_nist,sentence_nist"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OHTUatG7xMYj",
        "outputId": "6357709e-91c1-4d11-faef-33db9e91646a"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OAK27nMP40FD"
      },
      "source": [
        "# Part1 "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cH9AxD0t1UWj"
      },
      "source": [
        "## Preprocessing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JJ6gPUEzGVw1"
      },
      "source": [
        "### Load the dataset to colab directory "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wM8-U3jlllZU"
      },
      "source": [
        "For the sake of speed, copy the dataset files to the root directory of notebook and unziped them. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U6B4V0r_lUWR"
      },
      "source": [
        "!cp /content/drive/My\\ Drive/AFEC-merged-all.zip /content/AFEC-merged-all.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HHil6Sjp1GUH"
      },
      "source": [
        "!cp /content/drive/My\\ Drive/Test.zip /content/Test.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sDVCQP86ldOc",
        "outputId": "3d51a1b1-9e0f-4319-8ae7-184494234005"
      },
      "source": [
        "!unzip /content/AFEC-merged-all.zip"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/AFEC-merged-all.zip\n",
            "  inflating: AFEC-merged.en          \n",
            "  inflating: AFEC-merged.fa          \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EuHqCOVB5d-t"
      },
      "source": [
        "os.rename('/content/AFEC-merged.en',\"/content/AFEC-merged_en.txt\")\n",
        "os.rename('/content/AFEC-merged.fa',\"/content/AFEC-merged_fa.txt\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1C41P4jD1KU3",
        "outputId": "d0273977-c0d1-41eb-9090-1d309140c6ef"
      },
      "source": [
        "!unzip /content/Test.zip"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/Test.zip\n",
            "   creating: Test/\n",
            "  inflating: Test/test.en            \n",
            "  inflating: Test/test.fa0           \n",
            "  inflating: Test/test.fa1           \n",
            "  inflating: Test/test.fa2           \n",
            "  inflating: Test/test.fa3           \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JQxUKaRI515I"
      },
      "source": [
        "os.rename('/content/Test/test.en',\"/content/test_en.txt\")\n",
        "os.rename('/content/Test/test.fa0',\"/content/test_fa0.txt\")\n",
        "os.rename('/content/Test/test.fa1',\"/content/test_fa1.txt\")\n",
        "os.rename('/content/Test/test.fa2',\"/content/test_fa2.txt\")\n",
        "os.rename('/content/Test/test.fa3',\"/content/test_fa3.txt\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mm0HmkJyZDc7"
      },
      "source": [
        "### Vocabulary "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Nu-pqtNhZGt2"
      },
      "source": [
        "class Vocabulary_and_Dictionary():\n",
        "\n",
        "    def __init__(self, freq_threshold = 2):\n",
        "        self.freq_threshold = freq_threshold\n",
        "        \n",
        "        self.frequencies = {}\n",
        "        self.idx_2_str = {0: \"<pad>\", 1: \"<sos>\", 2: \"<eos>\", 3: \"<unk>\"}\n",
        "        self.str_2_idx = {\"<pad>\": 0, \"<sos>\": 1, \"<eos>\": 2, \"<unk>\": 3}\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.idx_2_str)\n",
        "\n",
        "    @staticmethod\n",
        "    def preprocess(text):\n",
        "        # prepare translation table for removing punctuation\n",
        "        table = str.maketrans('', '', string.punctuation)\n",
        "\n",
        "        # tokenize on white space\n",
        "        text = text.split()\n",
        "        # convert to lower case\n",
        "        text = [word.lower() for word in text]\n",
        "        # remove punctuation from each token\n",
        "        text = [word.translate(table) for word in text]\n",
        "\n",
        "        # remove tokens with numbers in them\n",
        "        text = [word for word in text if word.isalpha()]\n",
        "        \n",
        "        return text\n",
        "\n",
        "    def build_vocabulary_and_dictionary(self, text_file_root):\n",
        "\n",
        "        self.dic = {}\n",
        "        \n",
        "        idx = 4\n",
        "        lines = [line.strip() for line in open(text_file_root)]\n",
        "        \n",
        "        for i in range(len(lines)):\n",
        "\n",
        "            words =  self.preprocess(lines[i])\n",
        "            for word in words:\n",
        "                if word not in self.frequencies:\n",
        "                    self.frequencies[word] = 1\n",
        "\n",
        "                else:\n",
        "                    self.frequencies[word] += 1\n",
        "\n",
        "                if self.frequencies[word] == self.freq_threshold:\n",
        "                    self.str_2_idx[word] = idx\n",
        "                    self.idx_2_str[idx] = word\n",
        "                    idx += 1\n",
        "\n",
        "            self.dic[i] = [\" \".join(words)] \n",
        "\n",
        "    def IndexEncoding(self, text):\n",
        "        tokenized_text = self.preprocess(text)\n",
        "        index_encoded_text = [self.str_2_idx[token] if token in self.str_2_idx else self.str_2_idx[\"<unk>\"]\n",
        "                                for token in tokenized_text]\n",
        "        \n",
        "        return index_encoded_text\n",
        "\n",
        "    def IndexDecoding(self, sen_tensor):\n",
        "        tokenized_vec = list(sen_tensor.cpu().numpy())\n",
        "        decoded_vec = [self.idx_2_str[token] for token in tokenized_vec]\n",
        "        return decoded_vec\n",
        "\n",
        "    def get_str_2_idx(self):\n",
        "        return self.str_2_idx\n",
        "\n",
        "    def get_dic(self):\n",
        "        return self.dic"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KzLfXOlyJF4z",
        "outputId": "0fca3fb0-f832-46fb-cfc9-506efd3a85cc"
      },
      "source": [
        "en_vocab_dic = Vocabulary_and_Dictionary(freq_threshold = 10)\n",
        "\n",
        "train_text_root = \"/content/AFEC-merged_en.txt\"\n",
        "en_vocab_dic.build_vocabulary_and_dictionary(train_text_root)\n",
        "\n",
        "en_train_dic = en_vocab_dic.get_dic() \n",
        "\n",
        "print(len(en_vocab_dic.get_str_2_idx()))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "30032\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HOIEj5PTi3vp",
        "outputId": "1f0ea10f-d8b4-44c3-b88e-ab57d7e81541"
      },
      "source": [
        "en_train_dic[0],len(en_train_dic)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['north waziristan operation kills more militants'], 684417)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IpeczDWLB_MK",
        "outputId": "16934e7a-b85d-4f10-de1e-d96b91dbfd24"
      },
      "source": [
        "fa_vocab_dic = Vocabulary_and_Dictionary(freq_threshold = 10)\n",
        "\n",
        "train_text_root1 = \"/content/AFEC-merged_fa.txt\"\n",
        "fa_vocab_dic.build_vocabulary_and_dictionary(train_text_root1)\n",
        "\n",
        "fa_train_dic = fa_vocab_dic.get_dic() \n",
        "\n",
        "print(len(fa_vocab_dic.get_str_2_idx()))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "25256\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yKZhQ0BH595V",
        "outputId": "f69ac8ee-3e58-424a-b900-befb7e2de0ac"
      },
      "source": [
        "fa_train_dic[0],len(fa_train_dic)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['مرگ ستیزه جوی دیگر در عملیات وزیرستان شمالی'], 684417)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lcAeG6jMFbty",
        "outputId": "04faffec-d779-4e97-c5f8-85e51b565a80"
      },
      "source": [
        "en_vocab_dic2 = Vocabulary_and_Dictionary(freq_threshold = 2)\n",
        "\n",
        "test_text_root = \"/content/test_en.txt\"\n",
        "en_vocab_dic2.build_vocabulary_and_dictionary(test_text_root)\n",
        "\n",
        "en_test_dic = en_vocab_dic2.get_dic() \n",
        "\n",
        "print(len(en_vocab_dic2.get_str_2_idx()))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "203\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nphWhXA6Mdgu",
        "outputId": "3b6bb7ab-0619-4ccf-8142-a8c384c7f67a"
      },
      "source": [
        "fa_vocab_dic2 = Vocabulary_and_Dictionary(freq_threshold = 2)\n",
        "\n",
        "test_text_root0 = \"/content/test_fa0.txt\"\n",
        "fa_vocab_dic2.build_vocabulary_and_dictionary(test_text_root0)\n",
        "\n",
        "fa_test_dic0 = fa_vocab_dic2.get_dic() \n",
        "\n",
        "test_text_root1 = \"/content/test_fa1.txt\"\n",
        "fa_vocab_dic2.build_vocabulary_and_dictionary(test_text_root1)\n",
        "\n",
        "fa_test_dic1 = fa_vocab_dic2.get_dic() \n",
        "\n",
        "test_text_root2 = \"/content/test_fa2.txt\"\n",
        "fa_vocab_dic2.build_vocabulary_and_dictionary(test_text_root2)\n",
        "\n",
        "fa_test_dic2 = fa_vocab_dic2.get_dic() \n",
        "\n",
        "test_text_root3 = \"/content/test_fa3.txt\"\n",
        "fa_vocab_dic2.build_vocabulary_and_dictionary(test_text_root3)\n",
        "\n",
        "fa_test_dic3 = fa_vocab_dic2.get_dic() \n",
        "\n",
        "print(len(fa_vocab_dic2.get_str_2_idx()))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "491\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mxOiHqgNmaQj",
        "outputId": "49467012-31fb-43e9-84d0-cb4bd78dcd27"
      },
      "source": [
        "en_test_dic"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{0: ['hello do we drive together to hanover on the twentyeighth of march'],\n",
              " 1: ['it is more comfortable by train'],\n",
              " 2: ['do you go by car and i go by train'],\n",
              " 3: ['i would like to go by train and what would you like'],\n",
              " 4: ['if we take the ice train at six past seven we will arrive at twentyfive past eight'],\n",
              " 5: ['which cafe'],\n",
              " 6: ['the cafe at platform fourteen'],\n",
              " 7: ['in any case a cheap hotel'],\n",
              " 8: ['what did you say please'],\n",
              " 9: ['and how much is a single room'],\n",
              " 10: ['we can take a taxi from the station to the hotel'],\n",
              " 11: ['at which hotel do you want to reserve a room now and how much is a single room'],\n",
              " 12: ['okay should we drive back on friday evening'],\n",
              " 13: ['i think we rather drive back at thirtythree past nine then we will arrive at hamburg at fiftytwo past ten'],\n",
              " 14: ['fine and dont forget your swimming stuff maybe we can go swimming together'],\n",
              " 15: ['yes when and where do we want to meet'],\n",
              " 16: ['i prefer the plane'],\n",
              " 17: ['a good idea then we will meet at the airport tomorrow'],\n",
              " 18: ['no idea we will see it does not matter'],\n",
              " 19: ['good let us meet at nine oclock hopefully the plane wont be hijacked tomorrow'],\n",
              " 20: ['i have already booked two rooms at the grunschnabel'],\n",
              " 21: ['what did you say'],\n",
              " 22: ['yes we have two rooms at the grunschnabel i will reserve a taxi right now'],\n",
              " 23: ['what is planned for the evening'],\n",
              " 24: ['a good idea i have heard phantom of the opera is supposed to be played'],\n",
              " 25: ['fine i think we have arranged everything then we will meet tomorrow'],\n",
              " 26: ['hello we have to talk about our trip to hanover'],\n",
              " 27: ['right we will be at the expo two thousand in hanover on the fourth and fifth of september'],\n",
              " 28: ['i have already booked a flight'],\n",
              " 29: ['we will set off at a quarter past eight and arrive at hanover at twelve oclock'],\n",
              " 30: ['we will have to meet at the airport at seven oclock'],\n",
              " 31: ['we arrive at hanover at twelve oclock midday'],\n",
              " 32: ['fine which hotel do you have in mind'],\n",
              " 33: ['the intercontinental is my favourite hotel in hanover'],\n",
              " 34: ['thank you how do we go back again to hamburg'],\n",
              " 35: ['we will go back by train on the fifth of september'],\n",
              " 36: ['the best thing is we meet at the train station at eight oclock'],\n",
              " 37: ['exactly the train leaves hanover at six minutes after eight'],\n",
              " 38: ['we leave hanover at eight oclock and arrive at hamburg at half past nine'],\n",
              " 39: ['goodbye'],\n",
              " 40: ['i would prefer to fly'],\n",
              " 41: ['yes there is a flight at a quarter past nine'],\n",
              " 42: ['the plane arrives in hanover at twentyfive past twelve'],\n",
              " 43: ['i did not understand what you said about the hotel'],\n",
              " 44: ['yes would you please book two rooms'],\n",
              " 45: ['the single room costs a hundred deutschmarks did i understand you right'],\n",
              " 46: ['where is this hotel'],\n",
              " 47: ['when do we meet'],\n",
              " 48: ['yes would you please book this hotel'],\n",
              " 49: ['please repeat this once again'],\n",
              " 50: ['do you mean you always stay there'],\n",
              " 51: ['fine would you like to do something in the evening'],\n",
              " 52: ['unfortunately i did not understand you would you like to go to the cinema'],\n",
              " 53: ['would you like to go out for a meal afterwards'],\n",
              " 54: ['we still have to plan our return trip'],\n",
              " 55: ['when do we want to travel back to hamburg'],\n",
              " 56: ['could you repeat that once again please'],\n",
              " 57: ['yes i agree should we also fly back or go by train'],\n",
              " 58: ['there is a flight leaving hanover at a quarter past seven pm'],\n",
              " 59: ['fine shall i book this flight'],\n",
              " 60: ['goodbye'],\n",
              " 61: ['no we go on the twentythird'],\n",
              " 62: ['no i suggest to take the train in the morning'],\n",
              " 63: ['six oclock is too early i propose to take the train at ten oclock in the morning'],\n",
              " 64: ['do you agree'],\n",
              " 65: ['i did not understand that'],\n",
              " 66: ['i did not understand that again'],\n",
              " 67: ['i still have not understood anything'],\n",
              " 68: ['i suggest to stay at the hotel grunschnabel in hanover'],\n",
              " 69: ['yes please'],\n",
              " 70: ['so shall we take the hotel grunschnabel'],\n",
              " 71: ['well once again should we stay at the hotel grunschnabel'],\n",
              " 72: ['once again shall we stay at the hotel grunschnabel'],\n",
              " 73: ['yes that is a good idea'],\n",
              " 74: ['i did not understand that i suggest to go back to hamburg on the twentyfourth in the evening'],\n",
              " 75: ['yes i agree i propose to take the train again'],\n",
              " 76: ['i agree everything okay'],\n",
              " 77: ['hello should we plan our journey to the expo on october the ninth'],\n",
              " 78: ['i would prefer to take the train'],\n",
              " 79: ['we could leave hamburg at twentyfour past seven and we would be in hanover at thirtynine past eight'],\n",
              " 80: ['fine and where at the station'],\n",
              " 81: ['do you also want to reserve seats for us'],\n",
              " 82: ['are the single rooms cheap there'],\n",
              " 83: ['three hundred deutschmarks is a bit expensive isnt there anything cheaper'],\n",
              " 84: ['are there single rooms for eighty deutschmarks'],\n",
              " 85: ['fine but at which hotel do you want to reserve now'],\n",
              " 86: ['where is our hotel and what is its name'],\n",
              " 87: ['at which hotel do you want to reserve for us and where is the hotel'],\n",
              " 88: ['would you repeat the hotel name then please'],\n",
              " 89: ['what is the name of our hotel'],\n",
              " 90: ['in the evening i would like to relax and maybe go to the sauna what do you think'],\n",
              " 91: ['okay the program of the evening we can plan later on'],\n",
              " 92: ['i would like to take the train at eighteen past nine pm is that okay for you'],\n",
              " 93: ['fine see you later then'],\n",
              " 94: ['fine at what time would you like to leave'],\n",
              " 95: ['i would prefer the train'],\n",
              " 96: ['that is okay should we meet at the station at eight oclock then'],\n",
              " 97: ['okay do you want me to reserve a hotel for us'],\n",
              " 98: ['i would suggest the stadthotel because it is the cheapest one'],\n",
              " 99: ['the stadthotel'],\n",
              " 100: ['would you like a single room or a double room'],\n",
              " 101: ['okay should we go to the hotel by taxi immediately'],\n",
              " 102: ['would you like to do something in the evening'],\n",
              " 103: ['fine i agree with that'],\n",
              " 104: ['could you find out a train connection for wednesday evening'],\n",
              " 105: ['sorry when will we arrive at hamburg'],\n",
              " 106: ['that is okay then we can go back by taxi'],\n",
              " 107: ['okay goodbye'],\n",
              " 108: ['hello i call you to plan the journey to hanover with you'],\n",
              " 109: ['yes i would like to leave on wednesday the twentysixth of january'],\n",
              " 110: ['i prefer the plane'],\n",
              " 111: ['fine at what time should we leave'],\n",
              " 112: ['i propose to leave at nine oclock in the morning'],\n",
              " 113: ['where should we meet'],\n",
              " 114: ['at which office please'],\n",
              " 115: ['fine do you want me to reserve a hotel for us'],\n",
              " 116: ['the hotel is called prinzenhof and it is in the city center'],\n",
              " 117: ['do you agree on a double room'],\n",
              " 118: ['great i will reserve two single rooms'],\n",
              " 119: ['would you like to go out for a meal in the evening'],\n",
              " 120: ['fine i will choose a restaurant'],\n",
              " 121: ['i would like to leave on thursday afternoon'],\n",
              " 122: ['yes i agree with that'],\n",
              " 123: ['yes i agree that we leave at five oclock'],\n",
              " 124: ['i think we have talked about everything goodbye'],\n",
              " 125: ['should we meet at the expo in hanover in june'],\n",
              " 126: ['yes i thought we would go to hanover on the ninth and tenth of june'],\n",
              " 127: ['yes i have got a timetable when would you like to go'],\n",
              " 128: ['ahm there is a train leaving hamburg at twentyfour past ten it arrives at hanover at thirtynine past eleven'],\n",
              " 129: ['i agree then we could have breakfast together'],\n",
              " 130: ['i agree we need a hotel in hanover'],\n",
              " 131: ['unfortunately i did not understand you'],\n",
              " 132: ['in may on the eighteenth and nineteenth of may we are going together to hanover'],\n",
              " 133: ['i would like to ahm go by plane do you have a plan'],\n",
              " 134: ['i would like to fly and to be in hanover at about ten oclock'],\n",
              " 135: ['where should we meet'],\n",
              " 136: ['should we meet at the meeting point at the airport'],\n",
              " 137: ['would you please repeat that once again'],\n",
              " 138: ['i agree how can i recognize you'],\n",
              " 139: ['well when would you like to fly back'],\n",
              " 140: ['i agree when exactly does the plane take off'],\n",
              " 141: ['please repeat that once again'],\n",
              " 142: ['that sounds great ahm fine you arrange the flight'],\n",
              " 143: ['do you want me to ah book the hotel do you have any preferences'],\n",
              " 144: ['a single room and it is important for you to have a bar'],\n",
              " 145: ['i think there is a tv in all the hotels'],\n",
              " 146: ['there is a hotel near the fairground but it is very expensive'],\n",
              " 147: ['i agree a single room costs three hundred deutschmarks a night'],\n",
              " 148: ['okay i book the hotel'],\n",
              " 149: ['ahm should we meet on may the eighteenth in the evening'],\n",
              " 150: ['okay should we meet at the hotel at half past seven'],\n",
              " 151: ['fine we still have to arrange the ride from the airport to the hotel'],\n",
              " 152: ['i did not understand that'],\n",
              " 153: ['i suggest we take the subway to the hotel'],\n",
              " 154: ['the subway takes about twenty minutes to the station'],\n",
              " 155: ['i agree then we have to have dinner earlier'],\n",
              " 156: ['yes is everything okay so far then'],\n",
              " 157: ['yes well see you on the eighteenth then'],\n",
              " 158: ['hello i would like to go with you to the expo in hanover'],\n",
              " 159: ['i did not understand that would you prefer to fly or to drive'],\n",
              " 160: ['i would like to be in hanover until noon but i still do not know if you want to fly or to drive'],\n",
              " 161: ['that sounds very good where should we meet'],\n",
              " 162: ['yes fine how would half past seven suit you'],\n",
              " 163: ['fine i have a list of hotels for hanover'],\n",
              " 164: ['i would prefer a hotel in the city center then we can do something in the evening'],\n",
              " 165: ['unfortunately i did not understand that'],\n",
              " 166: ['i could not understand you again'],\n",
              " 167: ['fine then i propose the hotel prinzenhof that is situated in the city center'],\n",
              " 168: ['did you understand that i have proposed the hotel prinzenhof'],\n",
              " 169: ['fine i will book two single rooms for two hundred deutschmarks each'],\n",
              " 170: ['do you know a restaurant in hanover for i do not know any'],\n",
              " 171: ['i would really like to have chinese or japanese food but not italien food is that possible'],\n",
              " 172: ['that is fine then we still have to arrange the return journey do you have a schedule for a return flight'],\n",
              " 173: ['i have to be back in hamburg on friday at three oclock pm at the latest'],\n",
              " 174: ['i did not understand when the plane leaves hanover and when it is supposed to arrive at hamburg'],\n",
              " 175: ['i did not understand that yet'],\n",
              " 176: ['that is fine when does the plane leave hanover'],\n",
              " 177: ['then we arrive at the airport in hamburg again right'],\n",
              " 178: ['then everything is sorted out now then we will see each other on august the seventeenth'],\n",
              " 179: ['okay goodbye'],\n",
              " 180: ['good morning how are you'],\n",
              " 181: ['when should we go to hanover'],\n",
              " 182: ['fine let us meet at ten oclock in the morning should we go by train'],\n",
              " 183: ['thanks then we meet at the train station'],\n",
              " 184: ['when will we arrive in hanover'],\n",
              " 185: ['i have asked when we will arrive in hanover'],\n",
              " 186: ['fine i have already reserved a hotel'],\n",
              " 187: ['at the hotel maritim we can go there by taxi'],\n",
              " 188: ['yes i have already booked the hotel'],\n",
              " 189: ['i have booked two single rooms one for me and one for you'],\n",
              " 190: ['in the evening we should watch the newest movie with arnold schwarzenegger'],\n",
              " 191: ['fine let us meet at the station at nine oclock on thursday morning'],\n",
              " 192: ['what did you say please'],\n",
              " 193: ['yes we leave at nine oclock and arrive at hamburg at one oclock right'],\n",
              " 194: ['yes is everything cleared up then'],\n",
              " 195: ['then everything is arranged goodbye'],\n",
              " 196: ['hello i would prefer the intercontinental'],\n",
              " 197: ['fine let us take the intercontinental should we fly'],\n",
              " 198: ['no problem then let us meet where should we meet'],\n",
              " 199: ['yes we take the atlantichotel'],\n",
              " 200: ['thanks by the way we fly to hanover'],\n",
              " 201: ['yes our plane will take off at a quarter to seven and will land at hanover at ten past ten'],\n",
              " 202: ['fine by taxi we will be at the hotel within ten minutes'],\n",
              " 203: ['yes let us take two single rooms and what would you like to do in the evening'],\n",
              " 204: ['fine let us go out for a meal in the evening the best would be the grunschnabel'],\n",
              " 205: ['on wednesday we will go back by train to hamburg'],\n",
              " 206: ['we will leave at half past eight and will arrive at hamburg at ten oclock'],\n",
              " 207: ['the best thing would be for us to take the subway from our hotel to the station'],\n",
              " 208: ['fine let us go for a little walk in hamburg'],\n",
              " 209: ['i am looking forward to that goodbye'],\n",
              " 210: ['hello should we go by car to hanover'],\n",
              " 211: ['we should leave at ten oclock in the morning to be in hanover at twelve oclock'],\n",
              " 212: ['do you want me to book a hotel'],\n",
              " 213: ['i will book two single rooms with a shower at the konigshotel'],\n",
              " 214: ['yes i would suggest we fly back'],\n",
              " 215: ['yes i would suggest the flight at a quarter past seven'],\n",
              " 216: ['i agree'],\n",
              " 217: ['we will be back in hamburg at five past ten'],\n",
              " 218: ['yes we will be back at hamburg at five past ten would you like to go to the wienerwald in hanover in the evening'],\n",
              " 219: ['okay goodbye'],\n",
              " 220: ['mrs schmidt we still want to plan the journey to hanover'],\n",
              " 221: ['i would like to take the train on the thirtieth'],\n",
              " 222: ['during a flight i always get sick i would like to take the train'],\n",
              " 223: ['well i would like to take the train during a flight i get sick'],\n",
              " 224: ['please let us take the train'],\n",
              " 225: ['let us take the train at nine oclock'],\n",
              " 226: ['at six past nine then then let us meet at the station at half past eight'],\n",
              " 227: ['when do we arrive at hanover'],\n",
              " 228: ['that is very good at which hotel should we stay then what about the grunschnabel'],\n",
              " 229: ['i think we will take the metropol hotel could you reserve two single rooms'],\n",
              " 230: ['that is perfect would you like to go out for a meal in the evening'],\n",
              " 231: ['yes the steaks at the blockhouse are always very succulent'],\n",
              " 232: ['good idea we could take the one at the maschsee'],\n",
              " 233: ['that is perfect then we have talked about everything goodbye'],\n",
              " 234: ['hello we still have to plan our trip to hanover'],\n",
              " 235: ['i would like to fly that looks more important'],\n",
              " 236: ['the earliest flight leaves at five past eight should we take that one'],\n",
              " 237: ['the plane arrives at hanover at a quarter past eleven'],\n",
              " 238: ['let us meet at seven oclock we still have to check in'],\n",
              " 239: ['what do you think about the hotel gewandhaus'],\n",
              " 240: ['i would like one i would like to have one single room'],\n",
              " 241: ['you would like to have a room with tv and bar'],\n",
              " 242: ['i will book the hotelroom'],\n",
              " 243: ['we have already agreed on the hotel'],\n",
              " 244: ['unfortunately i did not understand that'],\n",
              " 245: ['let us leave at eight oclock pm there is a flight to hamburg going over milan isnt that great'],\n",
              " 246: ['let yourself get surprised'],\n",
              " 247: ['i could but i dont do it'],\n",
              " 248: ['at eight oclock pm'],\n",
              " 249: ['yes that is in the evening now we have talked about everything goodbye']}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Itzx5io7U7H8",
        "outputId": "b679fcfc-768b-4f40-a6c5-9ee817aa5db2"
      },
      "source": [
        "fa_vocab_dic2.get_str_2_idx()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'<eos>': 2,\n",
              " '<pad>': 0,\n",
              " '<sos>': 1,\n",
              " '<unk>': 3,\n",
              " 'آبدار': 55,\n",
              " 'آتلانتیک': 51,\n",
              " 'آرنولد': 113,\n",
              " 'آلمان': 120,\n",
              " 'آلمانی': 52,\n",
              " 'آن': 92,\n",
              " 'آنجا': 117,\n",
              " 'آگوست': 49,\n",
              " 'آیا': 6,\n",
              " 'اتاق': 22,\n",
              " 'اتاقهای': 27,\n",
              " 'اتاقی': 56,\n",
              " 'اتصال': 31,\n",
              " 'اتلانتیک': 66,\n",
              " 'اتومبیل': 5,\n",
              " 'اجازه': 199,\n",
              " 'اجرا': 14,\n",
              " 'ارزان': 118,\n",
              " 'ارزانتری': 30,\n",
              " 'ارزانترین': 34,\n",
              " 'از': 90,\n",
              " 'است': 5,\n",
              " 'استادهتل': 35,\n",
              " 'استدهتل': 137,\n",
              " 'استراحت': 54,\n",
              " 'استیکهای': 72,\n",
              " 'اسم': 53,\n",
              " 'اقامت': 45,\n",
              " 'الان': 13,\n",
              " 'ام': 165,\n",
              " 'اما': 182,\n",
              " 'امیدوارم': 14,\n",
              " 'ان': 19,\n",
              " 'انتخاب': 68,\n",
              " 'انجا': 28,\n",
              " 'انجام': 141,\n",
              " 'اهم': 44,\n",
              " 'اوم': 76,\n",
              " 'اپرا': 28,\n",
              " 'اکتبر': 47,\n",
              " 'اکسپو': 114,\n",
              " 'اگر': 9,\n",
              " 'اگوست': 59,\n",
              " 'ایا': 6,\n",
              " 'ایتالیایی': 102,\n",
              " 'ایده': 27,\n",
              " 'ایستگاه': 70,\n",
              " 'این': 84,\n",
              " 'اینترکانتیننتال': 205,\n",
              " 'اینجا': 97,\n",
              " 'اینکه': 69,\n",
              " 'با': 4,\n",
              " 'بار': 167,\n",
              " 'باز': 189,\n",
              " 'بازار': 39,\n",
              " 'بازگردیم': 20,\n",
              " 'بازگشت': 194,\n",
              " 'باشد': 185,\n",
              " 'باشم': 181,\n",
              " 'باشه': 132,\n",
              " 'باشید': 229,\n",
              " 'باشیم': 215,\n",
              " 'باهم': 19,\n",
              " 'باید': 28,\n",
              " 'بایستی': 24,\n",
              " 'ببخشید': 115,\n",
              " 'ببینیم': 114,\n",
              " 'بتوانیم': 18,\n",
              " 'بخورم': 58,\n",
              " 'بخوریم': 175,\n",
              " 'بخیر': 50,\n",
              " 'بد': 219,\n",
              " 'بده': 64,\n",
              " 'بدهید': 108,\n",
              " 'بدهیم': 103,\n",
              " 'برای': 47,\n",
              " 'برخواهیم': 34,\n",
              " 'برسد': 86,\n",
              " 'برمیگردیم': 9,\n",
              " 'برنامه': 50,\n",
              " 'بروم': 131,\n",
              " 'بروی': 91,\n",
              " 'بروید': 41,\n",
              " 'برویم': 139,\n",
              " 'برگردم': 47,\n",
              " 'برگردی': 35,\n",
              " 'برگردید': 80,\n",
              " 'برگردیم': 26,\n",
              " 'برگشت': 104,\n",
              " 'بریزیم': 134,\n",
              " 'بسیار': 8,\n",
              " 'بسیارخب': 48,\n",
              " 'بسیارخوب': 58,\n",
              " 'بعد': 56,\n",
              " 'بعدا': 135,\n",
              " 'بعدازآن': 12,\n",
              " 'بعدازظهر': 15,\n",
              " 'بفهمم': 44,\n",
              " 'بلاک': 127,\n",
              " 'بلند': 207,\n",
              " 'بله': 43,\n",
              " 'بمانیم': 108,\n",
              " 'بنابراین': 50,\n",
              " 'بنابرین': 33,\n",
              " 'به': 37,\n",
              " 'بهتر': 69,\n",
              " 'بهترین': 213,\n",
              " 'بود': 211,\n",
              " 'بکنی': 69,\n",
              " 'بگذار': 65,\n",
              " 'بگیرم': 32,\n",
              " 'بگیرید': 22,\n",
              " 'بگیریم': 104,\n",
              " 'بیا': 13,\n",
              " 'بیایم': 43,\n",
              " 'بیرون': 149,\n",
              " 'بیست': 12,\n",
              " 'تا': 18,\n",
              " 'تاکسی': 45,\n",
              " 'تر': 128,\n",
              " 'ترتیب': 31,\n",
              " 'ترجیح': 74,\n",
              " 'ترک': 72,\n",
              " 'تشخیص': 45,\n",
              " 'تلویزیون': 228,\n",
              " 'تماس': 63,\n",
              " 'تماشا': 63,\n",
              " 'تمام': 47,\n",
              " 'تنظیم': 116,\n",
              " 'تو': 7,\n",
              " 'توافق': 131,\n",
              " 'تکرار': 93,\n",
              " 'تکنفره': 122,\n",
              " 'جدیدترین': 111,\n",
              " 'جمعه': 196,\n",
              " 'جون': 70,\n",
              " 'حاضر': 58,\n",
              " 'حافظ': 136,\n",
              " 'حال': 57,\n",
              " 'حالا': 123,\n",
              " 'حالت': 60,\n",
              " 'حالم': 218,\n",
              " 'حداکثر': 105,\n",
              " 'حدود': 77,\n",
              " 'حرف': 87,\n",
              " 'حرکت': 151,\n",
              " 'خانم': 122,\n",
              " 'خب': 25,\n",
              " 'خدا': 135,\n",
              " 'خداحافظ': 96,\n",
              " 'خدانگهدار': 23,\n",
              " 'خواهد': 208,\n",
              " 'خواهم': 150,\n",
              " 'خواهید': 81,\n",
              " 'خواهیم': 56,\n",
              " 'خوب': 32,\n",
              " 'خوبه': 30,\n",
              " 'خوبی': 222,\n",
              " 'خود': 25,\n",
              " 'خودت': 76,\n",
              " 'خودمان': 145,\n",
              " 'خوی': 30,\n",
              " 'خیلی': 100,\n",
              " 'داتچمارک': 29,\n",
              " 'دارد': 81,\n",
              " 'دارم': 129,\n",
              " 'داری': 89,\n",
              " 'دارید': 64,\n",
              " 'داریم': 160,\n",
              " 'داشته': 227,\n",
              " 'در': 20,\n",
              " 'درباره': 15,\n",
              " 'درساعت': 6,\n",
              " 'درست': 198,\n",
              " 'درسته': 89,\n",
              " 'درطی': 52,\n",
              " 'درمورد': 35,\n",
              " 'دقیقا': 163,\n",
              " 'دقیقه': 13,\n",
              " 'ده': 102,\n",
              " 'دهم': 162,\n",
              " 'دهی': 20,\n",
              " 'دهید': 200,\n",
              " 'دهیم': 91,\n",
              " 'دو': 40,\n",
              " 'دوازده': 63,\n",
              " 'دوباره': 85,\n",
              " 'دوست': 10,\n",
              " 'دوش': 120,\n",
              " 'دوشزمارک': 27,\n",
              " 'دونفره': 146,\n",
              " 'دوهزار': 32,\n",
              " 'دویست': 56,\n",
              " 'دید': 22,\n",
              " 'دیگ': 26,\n",
              " 'دیگر': 107,\n",
              " 'را': 29,\n",
              " 'راجع': 152,\n",
              " 'راحتتر': 5,\n",
              " 'رانندگی': 179,\n",
              " 'راه': 18,\n",
              " 'ربع': 77,\n",
              " 'ربوده': 24,\n",
              " 'رزرو': 41,\n",
              " 'رستوران': 67,\n",
              " 'رستورانی': 8,\n",
              " 'رسید': 60,\n",
              " 'رفت': 33,\n",
              " 'رفتن': 49,\n",
              " 'رهسپار': 39,\n",
              " 'رو': 54,\n",
              " 'روز': 66,\n",
              " 'روی': 231,\n",
              " 'ریزی': 115,\n",
              " 'زمان': 38,\n",
              " 'زمانی': 136,\n",
              " 'زمین': 210,\n",
              " 'زوئن': 40,\n",
              " 'زود': 44,\n",
              " 'زودتر': 87,\n",
              " 'زودترین': 74,\n",
              " 'زیرا': 33,\n",
              " 'ساعت': 11,\n",
              " 'ساعتی': 38,\n",
              " 'سفر': 142,\n",
              " 'سفرمان': 113,\n",
              " 'سلام': 53,\n",
              " 'سمت': 21,\n",
              " 'سه': 195,\n",
              " 'سوار': 98,\n",
              " 'سواری': 90,\n",
              " 'سوم': 42,\n",
              " 'سونا': 55,\n",
              " 'سپتامبر': 68,\n",
              " 'سپس': 34,\n",
              " 'سکوی': 11,\n",
              " 'سی': 116,\n",
              " 'سیصد': 170,\n",
              " 'سینما': 40,\n",
              " 'شام': 88,\n",
              " 'شاید': 130,\n",
              " 'شبی': 84,\n",
              " 'شد': 209,\n",
              " 'شده': 204,\n",
              " 'شش': 71,\n",
              " 'ششم': 64,\n",
              " 'شما': 14,\n",
              " 'شنا': 16,\n",
              " 'شنایت': 11,\n",
              " 'شهر': 187,\n",
              " 'شوارزینگر': 62,\n",
              " 'شود': 29,\n",
              " 'شوم': 93,\n",
              " 'شوید': 133,\n",
              " 'شویم': 99,\n",
              " 'صبح': 103,\n",
              " 'صبحانه': 73,\n",
              " 'صحبت': 153,\n",
              " 'صد': 17,\n",
              " 'صرف': 101,\n",
              " 'صندلی': 48,\n",
              " 'طریق': 36,\n",
              " 'طول': 217,\n",
              " 'طی': 41,\n",
              " 'ظهر': 180,\n",
              " 'عازم': 66,\n",
              " 'عالی': 220,\n",
              " 'عرض': 9,\n",
              " 'عصر': 48,\n",
              " 'عصری': 61,\n",
              " 'غذا': 148,\n",
              " 'غذای': 98,\n",
              " 'فاصله': 42,\n",
              " 'فانتوم': 17,\n",
              " 'فراموش': 17,\n",
              " 'فردا': 38,\n",
              " 'فرودگاه': 62,\n",
              " 'فهمیدم': 18,\n",
              " 'فهمیدی': 46,\n",
              " 'فورا': 60,\n",
              " 'فکر': 31,\n",
              " 'فیلم': 112,\n",
              " 'قبلا': 110,\n",
              " 'قرار': 49,\n",
              " 'قطار': 8,\n",
              " 'قطاری': 71,\n",
              " 'قیمت': 23,\n",
              " 'قیمتی': 14,\n",
              " 'لطفا': 79,\n",
              " 'لیستی': 52,\n",
              " 'ما': 17,\n",
              " 'مارچ': 4,\n",
              " 'مارک': 119,\n",
              " 'ماریتیم': 109,\n",
              " 'ماشین': 214,\n",
              " 'ماه': 43,\n",
              " 'مایلم': 7,\n",
              " 'مایلید': 39,\n",
              " 'متاسفانه': 188,\n",
              " 'متاسفم': 32,\n",
              " 'مترو': 174,\n",
              " 'متروپل': 125,\n",
              " 'متشکرم': 201,\n",
              " 'متوجه': 36,\n",
              " 'مجبوریم': 22,\n",
              " 'محل': 78,\n",
              " 'مدت': 71,\n",
              " 'مرتب': 176,\n",
              " 'مرکز': 186,\n",
              " 'مریض': 123,\n",
              " 'مشکلی': 23,\n",
              " 'ملاقات': 35,\n",
              " 'ممکن': 193,\n",
              " 'من': 9,\n",
              " 'مناسب': 184,\n",
              " 'منتظر': 118,\n",
              " 'منظور': 37,\n",
              " 'منظورت': 21,\n",
              " 'مهم': 166,\n",
              " 'مهمتر': 73,\n",
              " 'موافقم': 110,\n",
              " 'موافقی': 24,\n",
              " 'موافقید': 147,\n",
              " 'مورد': 54,\n",
              " 'می': 172,\n",
              " 'میارزد': 85,\n",
              " 'میاید': 224,\n",
              " 'میبینم': 177,\n",
              " 'میبینیم': 52,\n",
              " 'میتوانستیم': 159,\n",
              " 'میتوانم': 79,\n",
              " 'میتوانی': 31,\n",
              " 'میتوانید': 61,\n",
              " 'میتوانیم': 133,\n",
              " 'میخواهی': 183,\n",
              " 'میخواهید': 124,\n",
              " 'میخواهیم': 156,\n",
              " 'میدادم': 206,\n",
              " 'میدهم': 75,\n",
              " 'میدهی': 46,\n",
              " 'میرسد': 225,\n",
              " 'میرسیم': 197,\n",
              " 'میروم': 5,\n",
              " 'میروی': 4,\n",
              " 'میروید': 6,\n",
              " 'میرویم': 75,\n",
              " 'میشناسی': 57,\n",
              " 'میشود': 127,\n",
              " 'میشوم': 124,\n",
              " 'میشویم': 43,\n",
              " 'میلان': 132,\n",
              " 'میمانی': 19,\n",
              " 'مینشیند': 36,\n",
              " 'میکند': 94,\n",
              " 'میکنم': 46,\n",
              " 'میکنی': 75,\n",
              " 'میکنید': 226,\n",
              " 'میکنیم': 83,\n",
              " 'میگیرم': 37,\n",
              " 'میگیریم': 67,\n",
              " 'نام': 128,\n",
              " 'نامش': 29,\n",
              " 'نتوانستم': 92,\n",
              " 'نخواهد': 12,\n",
              " 'ندارد': 28,\n",
              " 'ندارم': 21,\n",
              " 'نزدیک': 83,\n",
              " 'نشدم': 38,\n",
              " 'نشست': 117,\n",
              " 'نشود': 15,\n",
              " 'نظر': 164,\n",
              " 'نظری': 20,\n",
              " 'نفره': 24,\n",
              " 'نفهمیدم': 88,\n",
              " 'نمایشگاه': 46,\n",
              " 'نمیدانم': 51,\n",
              " 'نمیدهم': 134,\n",
              " 'نمیشناسم': 96,\n",
              " 'نه': 36,\n",
              " 'نهم': 154,\n",
              " 'نوزدهم': 74,\n",
              " 'نکن': 10,\n",
              " 'نکنید': 11,\n",
              " 'نیاز': 42,\n",
              " 'نیست': 121,\n",
              " 'نیم': 173,\n",
              " 'هامبورگ': 65,\n",
              " 'هانور': 53,\n",
              " 'هانوفر': 33,\n",
              " 'هانوور': 55,\n",
              " 'هتل': 19,\n",
              " 'هتلمان': 70,\n",
              " 'هتلها': 48,\n",
              " 'هجده': 57,\n",
              " 'هجدهم': 171,\n",
              " 'هر': 191,\n",
              " 'هرکدام': 55,\n",
              " 'هزار': 49,\n",
              " 'هزینه': 40,\n",
              " 'هست': 230,\n",
              " 'هستم': 119,\n",
              " 'هستند': 221,\n",
              " 'هشت': 59,\n",
              " 'هشتاد': 51,\n",
              " 'هشتم': 4,\n",
              " 'هفت': 61,\n",
              " 'هفدهم': 106,\n",
              " 'هم': 27,\n",
              " 'همدیگر': 129,\n",
              " 'همه': 111,\n",
              " 'همیشه': 216,\n",
              " 'همین': 13,\n",
              " 'هنوز': 105,\n",
              " 'هواپیما': 39,\n",
              " 'هواپیمای': 68,\n",
              " 'هیچ': 82,\n",
              " 'هیچی': 23,\n",
              " 'و': 7,\n",
              " 'واقع': 53,\n",
              " 'واقعا': 97,\n",
              " 'وجود': 168,\n",
              " 'وسایل': 10,\n",
              " 'وقت': 157,\n",
              " 'ولی': 192,\n",
              " 'وینروالد': 121,\n",
              " 'پرواز': 76,\n",
              " 'پریزنهاف': 190,\n",
              " 'پرینزنهاف': 94,\n",
              " 'پرینزنوف': 45,\n",
              " 'پس': 51,\n",
              " 'پنج': 78,\n",
              " 'پنجشنبه': 203,\n",
              " 'پنجم': 67,\n",
              " 'پیدا': 62,\n",
              " 'پیشنهاد': 101,\n",
              " 'چطور': 107,\n",
              " 'چقدر': 25,\n",
              " 'چه': 15,\n",
              " 'چهار': 158,\n",
              " 'چهارده': 12,\n",
              " 'چهاردهم': 7,\n",
              " 'چهارشنبه': 143,\n",
              " 'چهارم': 109,\n",
              " 'چون': 59,\n",
              " 'چگونه': 161,\n",
              " 'چی': 25,\n",
              " 'چیز': 112,\n",
              " 'چیزی': 8,\n",
              " 'چیست': 126,\n",
              " 'چینی': 99,\n",
              " 'ژانویه': 65,\n",
              " 'ژاپنی': 100,\n",
              " 'ژوئن': 155,\n",
              " 'کاری': 140,\n",
              " 'کافه': 10,\n",
              " 'کجا': 144,\n",
              " 'کجاست': 125,\n",
              " 'کجای': 26,\n",
              " 'کدام': 21,\n",
              " 'کرد': 73,\n",
              " 'کردم': 95,\n",
              " 'کردیم': 223,\n",
              " 'کمی': 50,\n",
              " 'کن': 37,\n",
              " 'کنم': 95,\n",
              " 'کنی': 178,\n",
              " 'کنید': 80,\n",
              " 'کنیم': 30,\n",
              " 'که': 86,\n",
              " 'کی': 82,\n",
              " 'گران': 169,\n",
              " 'گرانشابل': 44,\n",
              " 'گرانشنبل': 16,\n",
              " 'گرفت': 54,\n",
              " 'گرفتم': 41,\n",
              " 'گریچنبل': 26,\n",
              " 'گشت': 212,\n",
              " 'گفتی': 16,\n",
              " 'گفتید': 42,\n",
              " 'گواندهاس': 130,\n",
              " 'یا': 138,\n",
              " 'یازده': 72,\n",
              " 'یک': 16,\n",
              " 'یکبار': 106,\n",
              " 'یکدیگر': 34,\n",
              " 'یکنفره': 126,\n",
              " 'یکی': 202}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eqJa8-p8oruu"
      },
      "source": [
        "### AFEC Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "28RftuVnoyzi"
      },
      "source": [
        "class AFEC_Dataset(torch.utils.data.Dataset):\n",
        "    \"\"\"\n",
        "    Characterize AFEC Dataset for PyTorch. To be compatible with \n",
        "    DataLoader and other PyTorch syntaxes.\n",
        "    \"\"\"\n",
        "    def __init__(self, src_dic, trg_dic, indices, src_vocab, trg_vocab):\n",
        "        \n",
        "        self.src_dic = src_dic\n",
        "        self.trg_dic = trg_dic\n",
        "\n",
        "        # Get img names, caption columns\n",
        "        self.src_sentences = [self.src_dic[i] for i in indices]\n",
        "        self.trg_sentences = [self.trg_dic[i] for i in indices]\n",
        "\n",
        "        self.src_vocab = src_vocab\n",
        "        self.trg_vocab = trg_vocab \n",
        "            \n",
        "    def __len__(self):\n",
        "        'Denotes the total number of samples'\n",
        "        return len(self.src_sentences)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        \"\"\" \n",
        "        Generates one sample of data with label\n",
        "        \n",
        "        Arguments\n",
        "        ------------\n",
        "        index : integer, in range of len(self.list_IDs)\n",
        "                index of sample in list of IDs,\n",
        "        \n",
        "        Returns\n",
        "        ------------\n",
        "        X : torch tensor,\n",
        "            input data\n",
        "\n",
        "        Y : torch tensor,\n",
        "            correspond label of input data\n",
        "        \"\"\"\n",
        "        # Select sample\n",
        "        src_sentence = [ self.src_vocab.str_2_idx[\"<sos>\"] ]\n",
        "        src_sentence += self.src_vocab.IndexEncoding(self.src_sentences[index][0])\n",
        "        src_sentence.append(self.src_vocab.str_2_idx[\"<eos>\"])\n",
        "\n",
        "        trg_sentence = [ self.trg_vocab.str_2_idx[\"<sos>\"] ]\n",
        "        trg_sentence += self.trg_vocab.IndexEncoding(self.trg_sentences[index][0])\n",
        "        trg_sentence.append(self.trg_vocab.str_2_idx[\"<eos>\"])\n",
        "       \n",
        "        tensor_src_sentence = torch.tensor(src_sentence)\n",
        "        tensor_trg_sentence = torch.tensor(trg_sentence)\n",
        "\n",
        "        return tensor_src_sentence , tensor_trg_sentence"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qM1pU5x37fyR"
      },
      "source": [
        "### Split Dataset and Activate CUDA"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qT6heuOSrQU2"
      },
      "source": [
        "#train/val/test split \n",
        "random.seed(2045)\n",
        "\n",
        "indices = list(range(len(en_train_dic)))\n",
        "num_samples = len(en_train_dic)\n",
        "random.shuffle(indices)\n",
        "\n",
        "train_indices = indices[0 : int(0.9 * num_samples)]\n",
        "validation_indices = indices[int(0.9 * num_samples) : ]\n",
        "test_indices = list(range(len(en_test_dic)))\n",
        "\n",
        "partition = {'train' : train_indices , 'validation' : validation_indices,\n",
        "             'test' : test_indices }"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e4cMv5Z7rJ9F"
      },
      "source": [
        "# CUDA for PyTorch\n",
        "use_cuda = torch.cuda.is_available()\n",
        "device = torch.device(\"cuda:0\" if use_cuda else \"cpu\")\n",
        "torch.backends.cudnn.benchmark = True"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mIgDbPkxrs7c",
        "outputId": "8630826d-e4ec-4105-d9da-797de5f75eb0"
      },
      "source": [
        "device"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "device(type='cuda', index=0)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZK62oQqTx2TN",
        "outputId": "b65b3158-e3fc-4980-8651-6156e1cc9a18"
      },
      "source": [
        "len(partition['test'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "250"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y8W2Lo4NWaSn"
      },
      "source": [
        "### Padding"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XMkHPpjluZ1A"
      },
      "source": [
        "class BatchPadCollate:\n",
        "    def __init__(self, pad_idx):\n",
        "        self.pad_idx = pad_idx\n",
        "\n",
        "    def __call__(self, batch):\n",
        "\n",
        "        src_sentences = [item[0] for item in batch]\n",
        "        src_sentences = pad_sequence(src_sentences, batch_first= True, padding_value=self.pad_idx)\n",
        "\n",
        "        trg_sentences = [item[1] for item in batch]\n",
        "        trg_sentences = pad_sequence(trg_sentences, batch_first= True, padding_value=self.pad_idx) \n",
        "        \n",
        "        return src_sentences, trg_sentences "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gKfcsHtLDXHx"
      },
      "source": [
        "## Transformer Network "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SeEUzF5csGwu"
      },
      "source": [
        "class PositionalEncoding(nn.Module):\n",
        "    \n",
        "    def __init__(self,embed_dim, p_dropout, maximum_position_encoding):\n",
        "        super(PositionalEncoding, self).__init__()\n",
        "        \n",
        "        self.max_pos = maximum_position_encoding\n",
        "        self.dropout = nn.Dropout(p_dropout)\n",
        "        pos_enc = torch.zeros(self.max_pos, embed_dim)\n",
        "        positions = torch.arange(0, self.max_pos, dtype=torch.float).unsqueeze(1)\n",
        "        div_term = torch.exp(torch.arange(0, embed_dim, 2).float() * (-math.log(10000.0) / embed_dim))\n",
        "        pos_enc[:, 0::2] = torch.sin(positions * div_term)\n",
        "        pos_enc[:, 1::2] = torch.cos(positions * div_term)\n",
        "        pos_enc = pos_enc.unsqueeze(0).transpose(0, 1)\n",
        "        self.register_buffer('pos_enc', pos_enc)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = x + self.pos_enc[:x.size(0), :]\n",
        "        x = self.dropout(x)\n",
        "        \n",
        "        return x\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jAYwoKVODWWD"
      },
      "source": [
        "class _ScaledDotProductAttention(nn.Module):\n",
        "    \n",
        "    def __init__(self, embed_dim, device, p_dropout=0., kdim=None, vdim=None):\n",
        "        super(_ScaledDotProductAttention, self).__init__()\n",
        "        self.device = device\n",
        "        self.embed_dim = embed_dim\n",
        "        self.kdim = kdim if kdim is not None else embed_dim\n",
        "        self.vdim = vdim if vdim is not None else embed_dim\n",
        "\n",
        "        self.dropout = nn.Dropout(p_dropout)\n",
        "        self.softmax = nn.Softmax(dim=2)\n",
        "\n",
        "    def forward(self, query: Tensor, key: Tensor, value: Tensor, mask: Optional[Tensor] = None) -> Tuple[Tensor, Tensor]:\n",
        "          \n",
        "        \"\"\"\n",
        "        Calculate the attention weights.\n",
        "        q, k, v must have matching leading dimensions.\n",
        "        k, v must have matching penultimate dimension, i.e.: seq_len_k = seq_len_v.\n",
        "        The mask has different shapes depending on its type(padding or look ahead)\n",
        "        but it must be broadcastable for addition.\n",
        "\n",
        "        Args:\n",
        "            query : query shape == (..., seq_len_q, depth)\n",
        "            key : key shape == (..., seq_len_k, depth)\n",
        "            value: value shape == (..., seq_len_v, depth_v)\n",
        "            mask: Float tensor with shape broadcastable\n",
        "                to (..., seq_len_q, seq_len_k). Defaults to None.\n",
        "\n",
        "        Returns:\n",
        "            output, attention_weights\n",
        "        \"\"\"\n",
        "        qk = torch.matmul(query, torch.transpose(key,2,3))  # (..., seq_len_q, seq_len_k)\n",
        "        \n",
        "        # scale matmul_qk\n",
        "        scaled_attention_logits = qk / math.sqrt(self.kdim)\n",
        "\n",
        "        # add the mask to the scaled tensor.\n",
        "        if mask is not None:\n",
        "            scaled_attention_logits = scaled_attention_logits.masked_fill(mask == 1, -1e10)\n",
        "        \n",
        "        # softmax is normalized on the last axis (seq_len_k) so that the scores\n",
        "        # add up to 1.\n",
        "        attention_weights = self.softmax(scaled_attention_logits)  # (..., seq_len_q, seq_len_k)\n",
        "        \n",
        "        attention_weights = self.dropout(attention_weights)\n",
        "        \n",
        "        output = torch.matmul(attention_weights, value)  # (..., seq_len_q, depth_v)\n",
        "\n",
        "        return output"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gLx7hxqz5fIE"
      },
      "source": [
        "class _MultiHeadAttention(nn.Module):\n",
        "    \n",
        "    def __init__(self, embed_dim, num_heads, device, p_dropout=0.,\n",
        "                 kdim=None, vdim=None, epsilon = 1e-5):\n",
        "        super(_MultiHeadAttention, self).__init__()\n",
        "        \n",
        "        self.device = device \n",
        "        self.embed_dim = embed_dim                               \n",
        "        self.num_heads = num_heads\n",
        "        self.kdim = kdim * num_heads if kdim is not None else embed_dim\n",
        "        self.vdim = vdim * num_heads if vdim is not None else embed_dim\n",
        "        self.d_model = self.vdim \n",
        "        self.head_dim = self.d_model // num_heads\n",
        "        assert self.head_dim * num_heads == self.d_model, \"embed_dim must be divisible by num_heads\"\n",
        "\n",
        "        self.Wq = nn.Linear(embed_dim, self.kdim,device = device)\n",
        "        self.Wk = nn.Linear(embed_dim, self.kdim,device = device)\n",
        "        self.Wv = nn.Linear(embed_dim, self.vdim,device = device)\n",
        "\n",
        "        self.self_attention = _ScaledDotProductAttention(self.head_dim, device,p_dropout, kdim, vdim)\n",
        "        self.layernorm = nn.LayerNorm((embed_dim,), eps = epsilon, device = device)\n",
        "        self.linear = nn.Linear(self.vdim, embed_dim,device = device)\n",
        "        self.dropout = nn.Dropout(p_dropout)\n",
        "\n",
        "    def split_heads(self, x, batch_size):\n",
        "        \"\"\"\n",
        "        Split the last dimension into (num_heads, depth).\n",
        "        Transpose the result such that the shape is (batch_size, num_heads, seq_len, depth)\n",
        "        \"\"\"\n",
        "        x = x.view((batch_size, -1, self.num_heads, self.head_dim))\n",
        "        return torch.transpose(x,1,2)\n",
        "\n",
        "    def forward(self, query: Tensor, key: Tensor, value: Tensor, mask: Optional[Tensor] = None) -> Tuple[Tensor, Tensor]:\n",
        "        \n",
        "        batch_size = query.shape[0]\n",
        "\n",
        "        q = self.Wq(query)  # (batch_size, seq_len, d_model)\n",
        "        k = self.Wk(key)    # (batch_size, seq_len, d_model)\n",
        "        v = self.Wv(value)  # (batch_size, seq_len, d_model)\n",
        "\n",
        "        q = self.split_heads(q, batch_size)  # (batch_size, num_heads, seq_len_q, depth)\n",
        "        k = self.split_heads(k, batch_size)  # (batch_size, num_heads, seq_len_k, depth)\n",
        "        v = self.split_heads(v, batch_size)  # (batch_size, num_heads, seq_len_v, depth)\n",
        "\n",
        "        # scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)\n",
        "        # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)\n",
        "        scaled_attention = self.self_attention(q, k, v, mask)\n",
        "\n",
        "        scaled_attention = torch.transpose(scaled_attention, 1,2)  # (batch_size, seq_len_q, num_heads, depth)\n",
        "\n",
        "        concat_attention = scaled_attention.reshape((batch_size, -1, self.d_model))  # (batch_size, seq_len_q, d_model)\n",
        "\n",
        "        output = self.linear(concat_attention)  # (batch_size, seq_len_q, d_model)\n",
        "\n",
        "        output = self.dropout(output)\n",
        "        output = self.layernorm(value + output)\n",
        "        return output"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kd0VgGOSQkiy"
      },
      "source": [
        "class _PointwiseFeedicorward(nn.Module):\n",
        "    \n",
        "    def __init__(self, embed_dim, inner_dim,device, p_dropout=0.,epsilon = 1e-5):\n",
        "        super(_PointwiseFeedicorward, self).__init__()\n",
        "\n",
        "        self.linear1 = nn.Linear(embed_dim, inner_dim,device = device)\n",
        "        self.linear2 = nn.Linear(inner_dim, embed_dim,device = device)\n",
        "        self.layernorm = nn.LayerNorm((embed_dim,), eps = epsilon,device = device)\n",
        "        self.dropout = nn.Dropout(p_dropout)\n",
        "        self.relu = nn.ReLU()\n",
        "\n",
        "    def forward(self, x):\n",
        "        x1 = self.linear1(x)\n",
        "        x1 = self.relu(x1)\n",
        "        x1 = self.dropout(x1)\n",
        "        x1 = self.linear2(x1)\n",
        "        x1 = self.dropout(x1) \n",
        "        x1 = self.layernorm(x + x1)\n",
        "\n",
        "        return x1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eRM-2FoNQPWW"
      },
      "source": [
        "class _EncoderLayer(nn.Module):\n",
        "    \n",
        "    def __init__(self, embed_dim, inner_dim, num_heads, device, p_dropout = 0.,\n",
        "                 kdim = None, vdim = None,epsilon = 1e-5):\n",
        "        super(_EncoderLayer, self).__init__()\n",
        "\n",
        "        self.device = device\n",
        "        self.mha = _MultiHeadAttention(embed_dim, num_heads, device, p_dropout, kdim, vdim)\n",
        "        self.ff =  _PointwiseFeedicorward(embed_dim, inner_dim, device, p_dropout,epsilon = 1e-5)\n",
        "\n",
        "    def forward(self, x, mask = None):\n",
        "        \n",
        "        x = self.mha(x, x, x, mask)  # (batch_size, input_seq_len, d_model)\n",
        "        x = self.ff(x)         # (batch_size, input_seq_len, d_model)\n",
        "\n",
        "        return x"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OrNLrzhSSSJZ"
      },
      "source": [
        "class _TransformerEncoder(nn.Module):\n",
        "    \n",
        "    def __init__(self,num_layers, embed_dim,inner_dim, num_heads, device,\n",
        "                 p_dropout = 0., kdim = None, vdim = None, epsilon = 1e-5):\n",
        "        \n",
        "        super(_TransformerEncoder, self).__init__()\n",
        "\n",
        "        self.num_layers = num_layers \n",
        "        self.device = device\n",
        "        \n",
        "        kwrgs = {'p_droput': p_dropout, 'kdim': kdim, 'vdim': vdim, 'epsilon': epsilon}\n",
        "        # self.enc_layers = [_EncoderLayer(embed_dim, inner_dim, num_heads,device,**kwargs) for _ in range(num_layers)]\n",
        "        \n",
        "        self.enc_layers = nn.ModuleList([_EncoderLayer(embed_dim, inner_dim, num_heads,device,**kwargs) for _ in range(num_layers)])\n",
        "    def forward(self, x, mask):\n",
        "        \n",
        "        # for i in range(self.num_layers):\n",
        "        #     x = self.enc_layers[i](x, mask)\n",
        "        for layer in self.enc_layers:\n",
        "            x = layer(x,mask)\n",
        "        return x  # (batch_size, input_seq_len, d_model)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OIQ0HvlmnmDh"
      },
      "source": [
        "class Transformer(nn.Module):\n",
        "\n",
        "    def __init__(self, num_layers, embed_dim, ff_dim, num_heads, src_vocab_size,\n",
        "                 trg_vocab_size, device, p_dropout = 0., kdim = None,\n",
        "                 vdim = None, maximum_position_encoding = 5000, epsilon = 1e-5):\n",
        "        \n",
        "        super(Transformer, self).__init__()\n",
        "\n",
        "        self.device = device\n",
        "\n",
        "        self.src_embedding = nn.Embedding(src_vocab_size, embed_dim)\n",
        "        self.pos_encoder = PositionalEncoding(embed_dim, p_dropout, maximum_position_encoding)\n",
        "        self.scale = embed_dim\n",
        "        kwrgs = {'p_droput': p_dropout, 'kdim': kdim, 'vdim': vdim, 'epsilon': epsilon}\n",
        "        self.transformer_encoder = _TransformerEncoder(num_layers, embed_dim, ff_dim,\n",
        "                                                       num_heads,device = device, **kwargs)\n",
        "        \n",
        "\n",
        "        self.trg_embedding = nn.Embedding(trg_vocab_size, embed_dim)\n",
        "        self.pos_decoder = PositionalEncoding(embed_dim, p_dropout, maximum_position_encoding)\n",
        "        self.decoder_layer = TransformerDecoderLayer(embed_dim, num_heads, dim_feedforward = ff_dim,\n",
        "                                                     dropout = p_dropout, layer_norm_eps = epsilon, \n",
        "                                                     batch_first = True, device = device)\n",
        "        self.transformer_decoder = TransformerDecoder(self.decoder_layer, num_layers)\n",
        "\n",
        "\n",
        "        self.trg_classifier = nn.Linear(embed_dim, trg_vocab_size)\n",
        "\n",
        "    def make_padding_mask(self, seq, pad_idx):\n",
        "        seq_mask = seq == pad_idx\n",
        "        return seq_mask.to(self.device)\n",
        "    \n",
        "    def make_src_padding_mask(self, seq, pad_idx):\n",
        "        seq_mask = seq == pad_idx\n",
        "        return seq_mask.view(seq.shape[0],1,1,-1).to(self.device)\n",
        "\n",
        "    def generate_square_subsequent_mask(self, sz: int) -> Tensor:\n",
        "        r\"\"\"Generate a square mask for the sequence. The masked positions are filled with float('-inf').\n",
        "            Unmasked positions are filled with float(0.0).\n",
        "        \"\"\"\n",
        "        mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)\n",
        "        mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))\n",
        "        return mask.to(self.device)\n",
        "\n",
        "    def forward(self, src, trg):\n",
        "        \n",
        "        src_padding_mask = self.make_src_padding_mask(src, 0)\n",
        "        mem_padding_mask = self.make_padding_mask(src, 0)\n",
        "        src = self.src_embedding(src).to(self.device) * math.sqrt(self.scale)\n",
        "        src = self.pos_encoder(src).to(self.device)\n",
        "\n",
        "        enc_output = self.transformer_encoder(src, src_padding_mask) # (batch_size, src_seq_len, d_model)\n",
        "\n",
        "\n",
        "        trg_padding_mask = self.make_padding_mask(trg, 0)\n",
        "        trg_mask = self.generate_square_subsequent_mask(trg.shape[1])\n",
        "        kwargs = {'tgt_mask': trg_mask, 'tgt_key_padding_mask': trg_padding_mask, 'memory_key_padding_mask': mem_padding_mask}\n",
        "        trg = self.trg_embedding(trg).to(self.device) * math.sqrt(self.scale)\n",
        "        trg = self.pos_decoder(trg).to(self.device)\n",
        "\n",
        "        dec_output = self.transformer_decoder(trg, enc_output,**kwargs) # (batch_size, trg_seq_len, d_model)\n",
        "\n",
        "\n",
        "        final_output = self.trg_classifier(dec_output) # (batch_size, trg_seq_len, trg_vocab_size)\n",
        "\n",
        "        return final_output \n",
        "\n",
        "    # function to generate output sequence using greedy algorithm\n",
        "    def greedy_decode(self, src, max_len, sos_idx, eos_idx):\n",
        "        src = src.to(self.device)\n",
        "\n",
        "        # memory = self.encode(src).to(device)\n",
        "        y_pred = torch.ones(1, 1).fill_(sos_idx).type(torch.long).to(self.device)\n",
        "\n",
        "        for i in range(max_len-1):\n",
        "            \n",
        "            trg_mask = (self.generate_square_subsequent_mask(y_pred.size(1)).type(torch.bool)).to(self.device)\n",
        "            out = self.decode(src, y_pred, trg_mask)\n",
        "            # out = out.transpose(0, 1)\n",
        "            prob = self.trg_classifier(out[0, :, -1])\n",
        "            _, next_word = torch.max(prob, dim=0)\n",
        "            next_word = next_word.item()\n",
        "            y_pred = torch.cat([y_pred, torch.ones(1, 1).type_as(src.data).fill_(next_word)], dim=1)\n",
        "            if next_word == eos_idx:\n",
        "                break\n",
        "        return y_pred\n",
        "\n",
        "    def encode(self, src, mask):\n",
        "        src = self.src_embedding(src).to(self.device) * math.sqrt(self.scale)\n",
        "        src = self.pos_encoder(src)\n",
        "\n",
        "        enc_output = self.transformer_encoder(src, mask) # (batch_size, src_seq_len, d_model)\n",
        "        return enc_output\n",
        "    \n",
        "    def decode(self, trg, src_enc, trg_mask, mem_padding_mask):\n",
        "        kwargs = {'tgt_mask': trg_mask,'memory_key_padding_mask': mem_padding_mask}\n",
        "        \n",
        "        trg = self.trg_embedding(trg).to(self.device) * math.sqrt(self.scale)\n",
        "        trg = self.pos_decoder(trg)\n",
        "        # dec_output.shape == (batch_size, trg_seq_len, d_model)\n",
        "        dec_output = self.transformer_decoder(trg, src_enc, **kwargs)\n",
        "        final_output = self.trg_classifier(dec_output) # (batch_size, trg_seq_len, trg_vocab_size)\n",
        "\n",
        "        return final_output\n",
        "\n",
        "    def translate_sentence(self,src,max_len, sos_idx, eos_idx):\n",
        "        src = src.to(self.device)\n",
        "        src_padding_mask = self.make_src_padding_mask(src, 0)\n",
        "        mem_padding_mask = self.make_padding_mask(src, 0)\n",
        "\n",
        "        enc_src = self.encode(src, src_padding_mask)\n",
        "\n",
        "        trg_indexes = [sos_idx]\n",
        "\n",
        "        for i in range(max_len):\n",
        "\n",
        "            trg = torch.LongTensor(trg_indexes).unsqueeze(0).to(device)\n",
        "            trg_mask = self.generate_square_subsequent_mask(trg.shape[1])\n",
        "\n",
        "            output = self.decode(trg, enc_src, trg_mask, mem_padding_mask)\n",
        "            \n",
        "            pred_token = output.argmax(2)[:,-1].item()\n",
        "            \n",
        "            trg_indexes.append(pred_token)\n",
        "\n",
        "            if pred_token == eos_idx:\n",
        "                break\n",
        "        \n",
        "        return trg_indexes"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pneH1f0dHOau"
      },
      "source": [
        "## Sanity Check \n",
        "(try to overfit on a mini batch)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402,
          "referenced_widgets": [
            "328f9e84b17c4f8ba96f6818795f0771"
          ]
        },
        "id": "KvNcyRw6xhbG",
        "outputId": "1f76fdd8-e41e-48cf-8553-7610da18e782"
      },
      "source": [
        "torch.manual_seed(2045)\n",
        "\n",
        "pad_idx = 0\n",
        "\n",
        "# Parameters\n",
        "params = {'batch_size': 128,\n",
        "          'shuffle': True,\n",
        "          'num_workers':2,\n",
        "          'collate_fn' : BatchPadCollate(pad_idx=pad_idx)\n",
        "          }\n",
        "\n",
        "# Training and validation Data Generators\n",
        "training_set = AFEC_Dataset(en_train_dic,fa_train_dic,partition['train'],en_vocab_dic, fa_vocab_dic)\n",
        "training_generator = torch.utils.data.DataLoader(training_set, **params)\n",
        "\n",
        "validation_set =  AFEC_Dataset(en_train_dic,fa_train_dic,partition['validation'],en_vocab_dic, fa_vocab_dic)\n",
        "validation_generator = torch.utils.data.DataLoader(validation_set, **params)\n",
        "\n",
        "# Hyperparameters\n",
        "embed_dim = 256\n",
        "num_heads = 8\n",
        "dim_key = 64\n",
        "dim_value = 64\n",
        "dim_inner = 1024\n",
        "num_layers = 3\n",
        "src_vocab_size = len(en_vocab_dic.get_str_2_idx())\n",
        "trg_vocab_size = len(fa_vocab_dic.get_str_2_idx())\n",
        "learning_rate = 1e-4\n",
        "num_iterations = 400\n",
        "\n",
        "kwargs = {'p_dropout': 0.1,\n",
        "          'kdim': dim_key,\n",
        "          'vdim':dim_value,\n",
        "          }\n",
        "# Model\n",
        "model = Transformer(num_layers, embed_dim, dim_inner, num_heads, src_vocab_size,trg_vocab_size, device,**kwargs).to(device)\n",
        "\n",
        "# Define Loss function and Optimizer\n",
        "criterion = nn.CrossEntropyLoss(ignore_index = pad_idx)\n",
        "optimizer = optim.Adam(model.parameters(), lr = learning_rate, betas=(0.9, 0.98), eps=1e-9)\n",
        "\n",
        "train_losses = []\n",
        "validation_losses = []\n",
        "\n",
        "torch.autograd.set_detect_anomaly(True)\n",
        "\n",
        "train_dataiter = iter(training_generator)\n",
        "batch_train_src, batch_train_trg = train_dataiter.next()\n",
        "dataiter = iter(validation_generator)\n",
        "batch_src, batch_trg = dataiter.next() \n",
        "\n",
        "trainloss, valloss, running_trainloss, running_valloss = 0.0, 0.0, 0.0, 0.0\n",
        "\n",
        "# Loop over epochs\n",
        "for itr in tqdm(range(num_iterations)):\n",
        "\n",
        "    # Training        \n",
        "    # Transfer to GPU\n",
        "    batch_train_src, batch_train_trg = batch_train_src.to(device), batch_train_trg.to(device)\n",
        "\n",
        "    # zero the parameter gradients\n",
        "    optimizer.zero_grad()\n",
        "    \n",
        "    # Model computations\n",
        "    # forward + backward + optimize\n",
        "    batch_train_outputs = model(batch_train_src, batch_train_trg[: , :-1])\n",
        "    \n",
        "    loss = criterion(torch.transpose(batch_train_outputs,1,2), batch_train_trg[: , 1:])\n",
        "\n",
        "    loss.backward()\n",
        "\n",
        "    # Clip to avoid exploding gradient issues, makes sure grads are within a healthy range\n",
        "    nn.utils.clip_grad_norm_(model.parameters(), max_norm=2,error_if_nonfinite= False)\n",
        "\n",
        "    optimizer.step()\n",
        "\n",
        "    # print statistics\n",
        "    trainloss = loss.item()\n",
        "    running_trainloss += loss.item()\n",
        "\n",
        "    if itr % 100 == 99:    # print every 100 mini-batches\n",
        "        print(\"Cost of Train data after %i iterations : %f\" %(itr + 1, running_trainloss / 100))\n",
        "        running_trainloss = 0\n",
        "    \n",
        "    # Validation\n",
        "    with torch.set_grad_enabled(False):\n",
        "        # Transfer to GPU\n",
        "        batch_src, batch_trg = batch_src.to(device), batch_trg.to(device)\n",
        "        \n",
        "        # Model computations\n",
        "        # forward + backward + optimize\n",
        "        batch_outputs = model(batch_src, batch_trg[: , :-1])\n",
        "        \n",
        "        val_loss = criterion(torch.transpose(batch_outputs,1,2), batch_trg[: , 1:])\n",
        "\n",
        "        valloss = val_loss.item()\n",
        "        running_valloss += val_loss.item()\n",
        "\n",
        "        if itr % 100 == 99:    # print every 100 mini-batches\n",
        "            print(\"Cost of Validation data after %i iterations: %f\" %(itr + 1, running_valloss / 100))\n",
        "            running_valloss = 0\n",
        "\n",
        "    train_losses.append(trainloss)\n",
        "    validation_losses.append(valloss)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "328f9e84b17c4f8ba96f6818795f0771",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=400.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-34-6ba0f6c42eb3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     64\u001b[0m     \u001b[0;31m# Model computations\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m     \u001b[0;31m# forward + backward + optimize\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m     \u001b[0mbatch_train_outputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_train_src\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_train_trg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m     \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_train_outputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_train_trg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1049\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1050\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1052\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1053\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-33-8a4d68cf64c3>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, src, trg)\u001b[0m\n\u001b[1;32m     58\u001b[0m         \u001b[0mtrg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpos_decoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 60\u001b[0;31m         \u001b[0mdec_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransformer_decoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menc_output\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# (batch_size, trg_seq_len, d_model)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     61\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1049\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1050\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1052\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1053\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/transformer.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, tgt, memory, tgt_mask, memory_mask, tgt_key_padding_mask, memory_key_padding_mask)\u001b[0m\n\u001b[1;32m    246\u001b[0m                          \u001b[0mmemory_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmemory_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    247\u001b[0m                          \u001b[0mtgt_key_padding_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtgt_key_padding_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 248\u001b[0;31m                          memory_key_padding_mask=memory_key_padding_mask)\n\u001b[0m\u001b[1;32m    249\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    250\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnorm\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1049\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1050\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1052\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1053\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/transformer.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, tgt, memory, tgt_mask, memory_mask, tgt_key_padding_mask, memory_key_padding_mask)\u001b[0m\n\u001b[1;32m    403\u001b[0m         \"\"\"\n\u001b[1;32m    404\u001b[0m         tgt2 = self.self_attn(tgt, tgt, tgt, attn_mask=tgt_mask,\n\u001b[0;32m--> 405\u001b[0;31m                               key_padding_mask=tgt_key_padding_mask)[0]\n\u001b[0m\u001b[1;32m    406\u001b[0m         \u001b[0mtgt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtgt\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropout1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtgt2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    407\u001b[0m         \u001b[0mtgt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnorm1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtgt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1049\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1050\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1052\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1053\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/activation.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, query, key, value, key_padding_mask, need_weights, attn_mask)\u001b[0m\n\u001b[1;32m   1036\u001b[0m                 \u001b[0mtraining\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1037\u001b[0m                 \u001b[0mkey_padding_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkey_padding_mask\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mneed_weights\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mneed_weights\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1038\u001b[0;31m                 attn_mask=attn_mask)\n\u001b[0m\u001b[1;32m   1039\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_first\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1040\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mattn_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattn_output_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mmulti_head_attention_forward\u001b[0;34m(query, key, value, embed_dim_to_check, num_heads, in_proj_weight, in_proj_bias, bias_k, bias_v, add_zero_attn, dropout_p, out_proj_weight, out_proj_bias, training, key_padding_mask, need_weights, attn_mask, use_separate_proj_weight, q_proj_weight, k_proj_weight, v_proj_weight, static_k, static_v)\u001b[0m\n\u001b[1;32m   5082\u001b[0m     \u001b[0mattn_output\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattn_output_weights\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_scaled_dot_product_attention\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattn_mask\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdropout_p\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5083\u001b[0m     \u001b[0mattn_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mattn_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontiguous\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtgt_len\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbsz\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0membed_dim\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5084\u001b[0;31m     \u001b[0mattn_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mattn_output\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout_proj_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout_proj_bias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5085\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5086\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mneed_weights\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mlinear\u001b[0;34m(input, weight, bias)\u001b[0m\n\u001b[1;32m   1845\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhas_torch_function_variadic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1846\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mhandle_torch_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbias\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1847\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_nn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1848\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1849\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.7/traceback.py\u001b[0m in \u001b[0;36mformat_stack\u001b[0;34m(f, limit)\u001b[0m\n\u001b[1;32m    195\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mf\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    196\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getframe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf_back\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 197\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mformat_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mextract_stack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlimit\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlimit\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    199\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.7/traceback.py\u001b[0m in \u001b[0;36mextract_stack\u001b[0;34m(f, limit)\u001b[0m\n\u001b[1;32m    209\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mf\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    210\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getframe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf_back\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 211\u001b[0;31m     \u001b[0mstack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mStackSummary\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextract\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwalk_stack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlimit\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlimit\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    212\u001b[0m     \u001b[0mstack\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreverse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    213\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mstack\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.7/traceback.py\u001b[0m in \u001b[0;36mextract\u001b[0;34m(klass, frame_gen, limit, lookup_lines, capture_locals)\u001b[0m\n\u001b[1;32m    361\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlookup_lines\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    362\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mf\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 363\u001b[0;31m                 \u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    364\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    365\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.7/traceback.py\u001b[0m in \u001b[0;36mline\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    283\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    284\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_line\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 285\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_line\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlinecache\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlineno\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstrip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    286\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_line\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    287\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4Dx4tBb_6gqS",
        "outputId": "3de8709c-f1fe-4fef-c40b-d83892ecc532"
      },
      "source": [
        "# plot the cost\n",
        "plt.plot(train_losses)\n",
        "plt.plot(validation_losses)\n",
        "plt.title(\"Cost of Transfromer vs iters \")\n",
        "plt.ylabel('Cost')\n",
        "plt.xlabel('Iterations')\n",
        "plt.legend(labels = [\"Train\",\"Validation\"])\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEWCAYAAABhffzLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3gVVfrA8e+bTjophECAJPTeQpWq2FEsoKKrIhawrrqrq667uu23q+uuZcXeFURFRVQUK9J7r9JLSCiBFEhPzu+PM8EQkxAgySS57+d57pN7p533TpJ3zpw5c0aMMSillPIcXm4HoJRSqnZp4ldKKQ+jiV8ppTyMJn6llPIwmviVUsrDaOJXSikPo4lf1QkicpaIbBGRoyJymdvxlCUiMSIyR0SyROQ/bsdTG0SkpfP78HY7FlW9NPF7IBG5VkSWOf/UKSLylYgMOsNt7hSREWewib8Czxtjgo0x08ts+2ipV7GI5JT6fN2ZxH0KbgMOAaHGmN/VUpmuMsbsdn4fRQAiMltEbnE7LnXmNPF7GBG5H3gG+D8gBmgJvACMcjMuoBWwvrwZTvIJNsYEA7uBS0pNm1yynIj41HB8G0wFdzzWcNmVcrPsU6FnDnWIMUZfHvICwoCjwJhKlvHHHhj2Oa9nAH9nXhTwBZAOHAbmYisP7wLFQI6z/Qcr2PatwFZn3RlAM2f6tjLr+1cS305ghPN+GLAX+AOQ6sTR2InxIHDEeR9Xav3ZwN+A+UAW8A0Q5cwLAN4D0pzvuBR7cHwLKADynfhGAI8D05zlM4FbgGbO9zrsfM9bS5X7OPCRs3wWsBZoBzwMHAD2AOeV+V29DqQAycDfAW9n3jgn/qedWP9eZh81c/ZlRKlpPbFnLL5AG+AnIMOZ9kEF+zoeMIAP8A+gCMh19sHzzjIdgG+d77wZuKrU+m8BLwIzgWPOfrsI2ODsg2Tg927/X3jiy/UA9FWLv2y4ACgEfCpZ5q/AIqAJEA0sAP7mzPsn8JKTPHyBwYA483biJOQKtnu2k2R6YQ8u/wPmlJpf6frlLYdN/IXAE842GwGRwJVAIBDiJNvppdafjT3QtHOWnw38y5k3AfjcWdcb6I1t2ilJYn8vtZ3HsQeDy7AHv0bAHOzZUwDQA3vwObvU8rnA+U4ifQfYAfzR2Ze3AjtKbf9T4GUgyPldLAEmOPPGOd/7bmdbjcrZTz9w4oHn38BLzvv3nXK9nFgHVbCv43ESf6l9d0up+UHYA9ZNThwlB5dOpfZZBnBWqbJSgMHO/MZAL7f/LzzxpU09niUSOGSMKaxkmeuAvxpjDhhjDgJ/Aa535hUAsUArY0yBMWaucf6Dq+A64A1jzApjTB62pjtAROJP54uUUgw8ZozJM8bkGGPSjDEfG2OyjTFZ2Jrq0DLrvGmM+dkYkwN8iE3SJd8vEmhjjCkyxiw3xmRWUvZCY8x0Y0wx9mzoLOAPxphcY8wq4DXghlLLzzXGzHL2/0fYA+u/jDEFwFQgXkTCRSQGWzO+1xhzzBhzAFu7v6bUtvYZY/5njCl0vkdZU4CxACIizrpTSn3PVtgzrlxjzLxKvmNlRgI7jTFvOnGsBD4GxpRa5jNjzHxjTLExJtcpu5OIhBpjjhhjVpxm2eoMaOL3LGlA1EnahJsBu0p93uVMA1tr3Ap8IyLbReShUyj7hO0aY4468TQ/hW2U56CTUAAQkUAReVlEdolIJrYWHl6mfTm11PtsINh5/y4wC5gqIvtE5EkR8a2k7D2l3jcDDjsHmxK7OPH77S/1Pgd7EC4q9RknllbYs4AUEUkXkXRs7b9JBWWX52PsgTUWGII9QM515j0ICLBERNaLyPiTbKsirYB+JTE6cV4HNK0kziuxB7VdIvKTiAw4zbLVGdDE71kWAnnY5omK7MP+Q5do6UzDGJNljPmdMSYRuBS4X0TOcZY7Wc3/hO2KSBC2dp18St/g18qW+zugPdDPGBOKTXpgE13lG7JnMX8xxnQCBmJrtDdUtkqp9/uACBEJKTWtJaf3/fZgf09Rxphw5xVqjOlcQdm/DsyYI9jrF1cD1wJTS87OjDGpxphbjTHNsM1bL4hImyrEVbbMPcBPpWIMN/aC++0VrWOMWWqMGYU9iE3HnnGpWqaJ34MYYzKAPwOTROQyp3bsKyIXisiTzmLvA4+KSLSIRDnLvwcgIiNFpI3TdJCBvdhX7Ky3H0ispPj3gZtEpIeI+GN7FS02xuys5q8Zgq09p4tIBPBYVVcUkeEi0tU5O8jENksUn2Q1AIwxe7DXQ/4pIgEi0g24GWffnQpjTAo2af9HREJFxEtEWotI2Sark5mCPXCN5pdmHkRkjIjEOR+PYJNzVb5n2d/xF0A7Ebne+TvyFZE+ItKxvJVFxE9ErhORMKd5K7OK5apqponfwxhj/gPcDzyKvfi4B7gLW/sC23tkGbAG2/NkhTMNoC3wHbZXx0LgBWPMj868f2IPGOki8vtyyv0O+BO2CSIFaM2JbdbV5RnshdZD2IvUX5/Cuk2xPXUygY3Yni/vnsL6Y7EXRPdhL84+5nzv03ED4IftAXPEiSv2FLcxA/s7SzXGrC41vQ+wWESOOsv81hizvQrbexYYLSJHROQ5p1nrPOzvcR+2Ca3kQntFrgd2Os1wE7FNQ6qWlfTIUEop5SG0xq+UUh5GE79SSnkYTfxKKeVhNPErpZSHqReDO0VFRZn4+Hi3w1BKqXpl+fLlh4wx0WWn14vEHx8fz7Jly9wOQyml6hUR2VXedG3qUUopD6OJXymlPIwmfqWU8jD1oo1fKdVwFBQUsHfvXnJzc0++sKqSgIAA4uLi8PWtbDDZX2jiV0rVqr179xISEkJ8fDx2vD91JowxpKWlsXfvXhISEqq0jjb1KKVqVW5uLpGRkZr0q4mIEBkZeUpnUJr4lVK1TpN+9TrV/dmgE/9nq5J5b1G53ViVUspjNejEP2t9KpN+3IoOPa2UKpGWlkaPHj3o0aMHTZs2pXnz5sc/5+fnV7rusmXLuOeee2op0prToC/unttS2LNuA9sO9qVNk5CTr6CUavAiIyNZtWoVAI8//jjBwcH8/ve/PDuosLAQH5/yU2NSUhJJSUm1EmdNatA1/gs3PsS/fV9m7pZDboeilKrDxo0bx8SJE+nXrx8PPvggS5YsYcCAAfTs2ZOBAweyefNmAGbPns3IkSMBe9AYP348w4YNIzExkeeee87Nr3BKGnSNP6DHGDrs+z1T1i+Bs6rWzUkpVXv+8vl6NuzLrNZtdmoWymOXdD75gmXs3buXBQsW4O3tTWZmJnPnzsXHx4fvvvuORx55hI8//vhX62zatIkff/yRrKws2rdvz+23317lvvRuatCJn85XUDTzD7RI/pL8wjH4+TToExyl1BkYM2YM3t7eAGRkZHDjjTeyZcsWRISCgoJy17n44ovx9/fH39+fJk2asH//fuLi4spdti5p2Ik/KJLDTQdxUco8lu9MY0CbX41OqpRy0enUzGtKUFDQ8fd/+tOfGD58OJ9++ik7d+5k2LBh5a7j7//Lc+W9vb0pLCys6TCrRYOvAof0GUtzSWPT4lluh6KUqicyMjJo3rw5AG+99Za7wdSABp/4A7peSrZXEM22TqGoWLt1KqVO7sEHH+Thhx+mZ8+e9aYWfyqkPvRxT0pKMmfyIJbt795Ni62TWTtmPr26dKzGyJRSp2rjxo107Kj/h9WtvP0qIsuNMb/qf9rga/wATUfcha8UkT7vVbdDUUop13lE4g+Mbc/6wL50T/2Yorxst8NRSilXeUTiB8jsfSeRpLP7u5fcDkUppVzlMYm/+6CLWWo6ELHqBSjMczscpZRyjcck/kB/XxY1v5mwgoMUL3vT7XCUUso1NZb4ReQNETkgIutKTYsQkW9FZIvzs3FNlV+exH4jWVjUiaIfn4Dsw7VZtFJK1Rk1WeN/C7igzLSHgO+NMW2B753PtebsjjE87XMTkpcJn98D9aArq1Kqeg0fPpxZs068ofOZZ57h9ttvL3f5YcOGUdKd/KKLLiI9Pf1Xyzz++OM89dRTlZY7ffp0NmzYcPzzn//8Z7777rtTDb9a1FjiN8bMAcpWq0cBbzvv3wYuq6nyy9PIz5v+A4by78IxsPFzWPH2yVdSSjUoY8eOZerUqSdMmzp1KmPHjj3pujNnziQ8PPy0yi2b+P/6178yYsSI09rWmartNv4YY0yK8z4ViKnl8rlhYDxvcwlbgnrDVw/BwZ9rOwSllItGjx7Nl19+efyhKzt37mTfvn28//77JCUl0blzZx577LFy142Pj+fQITvM+z/+8Q/atWvHoEGDjg/bDPDqq6/Sp08funfvzpVXXkl2djYLFixgxowZPPDAA/To0YNt27Yxbtw4pk2bBsD3339Pz5496dq1K+PHjycvL+94eY899hi9evWia9eubNq0qVr2gWuDtBljjIhU2NYiIrcBtwG0bNmy2sqNCvZndFJLblp6M3NCHsXr4/Fw87fg26jaylBKVdFXD0Hq2urdZtOucOG/KpwdERFB3759+eqrrxg1ahRTp07lqquu4pFHHiEiIoKioiLOOecc1qxZQ7du3crdxvLly5k6dSqrVq2isLCQXr160bt3bwCuuOIKbr31VgAeffRRXn/9de6++24uvfRSRo4cyejRo0/YVm5uLuPGjeP777+nXbt23HDDDbz44ovce++9AERFRbFixQpeeOEFnnrqKV577bUz3kW1XePfLyKxAM7PAxUtaIx5xRiTZIxJio6u3lE1Jw5tTappzPvNHoTUdfDROCgqf9hVpVTDU7q5p6SZ58MPP6RXr1707NmT9evXn9AsU9bcuXO5/PLLCQwMJDQ0lEsvvfT4vHXr1jF48GC6du3K5MmTWb9+faWxbN68mYSEBNq1awfAjTfeyJw5c47Pv+KKKwDo3bs3O3fuPN2vfILarvHPAG4E/uX8/KyWywcgrnEgo3vH8ZcVwsjz/0XYD3+ATyfAFa+Cl7cbISnlmSqpmdekUaNGcd9997FixQqys7OJiIjgqaeeYunSpTRu3Jhx48aRm5t7WtseN24c06dPp3v37rz11lvMnj37jGItGfq5Ood9rsnunO8DC4H2IrJXRG7GJvxzRWQLMML57Io7h7fBYHhsXz8Y8RdY9zF8cS8UF7sVklKqlgQHBzN8+HDGjx/P2LFjyczMJCgoiLCwMPbv389XX31V6fpDhgxh+vTp5OTkkJWVxeeff358XlZWFrGxsRQUFDB58uTj00NCQsjKyvrVttq3b8/OnTvZunUrAO+++y5Dhw6tpm9avhqr8RtjKrpEfk5NlXkqWkQEcvuwNjz3/RZG9byW4YOzYO5Ttn//5S+Bvz6cXamGbOzYsVx++eVMnTqVDh060LNnTzp06ECLFi0466yzKl23V69eXH311XTv3p0mTZrQp0+f4/P+9re/0a9fP6Kjo+nXr9/xZH/NNddw66238txzzx2/qAsQEBDAm2++yZgxYygsLKRPnz5MnDixZr60wyOGZa5IXmERFz07l5z8Ir66ZzBha16Fb/4EkW3gmikQ1abay1TK0+mwzDVDh2WuIn8fb/5zVQ8OZOXxpxnrYcCdcP2nkH0IXh0Om792O0SllKp2Hp34AXq0COeus9swY/U+fvr5ICQOhdtmQ+N4eP8a+OlJbfdXSjUoHp/4wXbvbNMkmHveX8mutGMQ3hJu/ga6XQU//gPevxqOVtjzVCl1iupDE3N9cqr7UxM/EODrzRs39sEYw11TVpJXWGRv6Lr8ZbjoKdgxB14YoE0/SlWDgIAA0tLSNPlXE2MMaWlpBAQEVHkdj764W9as9alMeHc54wbG8/ilnX+ZcWATfHKLvcNwwF0w7CHt9aPUaSooKGDv3r2n3U9e/VpAQABxcXH4+vqeML2ii7uuDdlQF53fuSk3D0rg9Xk76NA0hGv6OkNFNOlgh3X46g+wcBJs/Q5GPgOtBrgbsFL1kK+vLwkJCW6H4dG0qaeMhy/swJB20Tw6fR2Ltqf9MsO3EVz6HNwwHXIz4c0LbdfPwnz3glVKqdOgib8MH28v/je2J60iA7n9veXsTivzcPbEYXD3Mug9DhY8B6+PgP2Vj8WhlFJ1iSb+coQ18uX1G/tggJvfXkpmbpkB3PyC4JJn4OrJkLEXXh4C3/8NCrTNUilV92nir0B8VBAvXNeLHYeOMe6NJWTklDN6Z8eRcOdS6DrGDvfw0iDYtbD2g1VKqVOgib8SA1tH8fy1PVmbnMHt7y0nv7CcG7mCIu3YPr/5BArz4M0L4MvfQ96vB2NSSqm6QBP/SVzQJZYnR3djwbY0/vjp2or7Hrc5B+5YCP0mwtLXYFJ/7fevlKqTNPFXweU947jnnLZ8tHwvL8zeVvGC/sFw4RP2rl//YHvH7wfXQ+a+2gtWKaVOQhN/Fd03oi2jejTj37M2M2P1SRJ5i74wYS6c82fY8g083xdm/dEO+ayUUi7TxF9FIsKTo7vRNyGC3324igVbD1W+go8fDP4d3LEI2p0Hi16E//WGpa/rYx6VUq7SxH8K/H28efX6JBKigpjw7nI2pmSefKWIBBj9BkycB006wpf3w6S+sHaajvqplHKFJv5TFBboy1s39SXI34eb3lxKSkZO1VaM6QTjvoRrPwSfRvDxzbb75/rpegBQStUqTfynoVl4I968qQ9H8woZ/9Yyssre4FUREWh3vq39X/EaFOXDRzfaM4CFL+gNYEqpWqGJ/zR1jA1l0nW9+Hl/Fre/t6L8Pv4V8fKCbmPgzsVw5esQGAmzHobnk+wgcDnpNRe4UsrjaeI/A0PbRfOvK7oyb+shHpy2muLiUxzi2ssbuo6Gm2fBDTMgLA5mPQL/7QRf3GeHg1ZKqWqmwzKfoTFJLTiQlce/Z20mJjSAhy86zYdIJw61r5TVsPgVWDkZlr0Bbc+D4Y9AbA/bVKSU8hyHt0NEYrVvVhN/NbhjWGtSM3J5ec52YkIDGD/oDMYaj+0Ol02Cc/8Ky9+Eec/AK8Mgsi30ugF6/gYCI6otdqXUGTCm/ArZ0YN2elAUHEuDtK2QvhvSd0FWKsR0Bm9f2L0IfANt9++cdPv+yA5I3wM5R+DoftslvEmHag1bn8BVTYqKDXdOXsGsDan8b2xPRnZrVj0bzj4MGz+HVVNgzyLwD7PXB+L6QIeR9g5hpdTpyzv66/+jghwQL/Dxh9wMWPGObYrtcAksfRUWvwT+oTahxyXZjhnpu6FpVzj0sx211xRBSCxkpZy4bd9AKHCGe/cLBlNsy/NtZKcFN7Flh8VB67Ohzy2n/cS/ip7ApYm/GuUWFHH964tZvSeDt8b3YWDrqOotIHUtzP2PHQOoMAf8Qmwvod7joOUA8NYTOOWhigpsDbq04mLIP2qT85Edtht1aDPIPwZ+gbZWPu+/sHshdLoMMDYRH90PuxbYhOztb//XipwHLom3TejNetpOGQjsXQJevhA/yDbVRiRAeCsIaw4HN0NMF2jSCcJbQngL8AmA1DX2wBEcYz8XHLOfq7k5VxN/LcnILmD0SwtIzcjlw4kD6BgbWv2FFBfbP7aV78GmL+wpoX8oxA+G1sPtw2Ii2+g1AVW/FRXA5pmQssbe/NjuAigugPnPgm8QNG5lE/ummTZ5Jw6DIztt8k5ZbRP2yQRGQfNesGMu+AbYJ+qFNLUHiKi24OVjE3Pny2HXfNj6PfS41g7FXvr/q6ImH5dp4q9F+9JzuOKFBRQbwyd3DCSucWDNFZZ/zI4HtO1H2P6jPd0ECI2DhCH2jzq2h21T9KvBOJQqkZVqKyTh8RCZCId32LbtxvEQ0xWSl0FmMqRtg4BwOLTZvvcPscn2yC57dluQA/kVDW8ugJO7vP1tLbso31aA8rLsmXBAmG0+iUi0r6J8u22fAHsACYy0/xcBYc4wKtLgzpo18deyzalZjHlpAdEh/kybOJDGQX61U/Dh7bB9tj0Q7JoP2c5zg8XLXiBu2hWadnF+drPtiUqVyEm3o8lGJMK+lRDZ2ibFYwdtwszLgpXvQqNwm5ibdLJt2lu+sU0ZEa3t/MJTuBkxrCVEt7NlH94GjRrbpsusFNuZoeOltull1wJbw4/rY3vAZe2HgFBba/eppf+vekYTvwsWb0/j+jeW0LV5GJNv6UeAr3ftBmCMrVntW2VPffevszWpjD2/LBMc4xwEutq2yKbd7D+7Vy3Hqk5fQY595RyB5BW2mSK8pa1571lsLzT6BUFRnk3Mh7dBZoqdX1xgE2xkG9umvWuB055dqkZdEd8g2zbt5QutBtp265x02wwy6D5I22IPClHtoHGCfX9kpx29trjItonDr9vmVbXRxO+Sr9amcMeUFYzoGMOL1/XCx7sO3DOXcwRS1/1yIEhdY28WK3aGnvBpBMHRtgdReAsoLoQW/ezZgbcfhDa3PRlyM2yvh4DwOtm+WWcYY5NscIzdx+m77ZlYcIxNtoV5trkjfbetVecftbXggmO29u0fYptCDm2xNdz962wt3Bib4FNWO8m6gv/lkgTt7WeXE+dvMKK17RpclA/719smj9bn2Lbyg5tsu/rR/Xb5iETbrGiMbUbx9rOxZuyGoGh7YCkust9FmxTrDE38Lnp7wU4em7Gea/u15B+XdUHqYpIszLfJJ3WtPShkp0H2IVszLC608yriE2BrjI0a2zMFv2D7Cm1mPwdG2VP/gmx7ZtE4wbb5gu2yVtKdrSDHLhcYZbe3a559HxBq44nra9ts8zJtLTashZ0HNiEV5kJ+tq1B+ofY5omiAjiw0X4+ssPOa5wAGJswvf1s2zNiE1jBMXtAO7TFTs8/avdBXqZ9v2Ou7Zvd6iy7/NqPbGyRreHoAbsdU2zLLsyz5fgE2Fq2T8Av06rMqXn7BtoyCvPsmZmXt/2+2UegWQ+nPTvQ1qbTd9tE3jge2l9k4y0usgk8Y7fdp17ev3QfBNthwKsOVEpUtapTiV9E7gNuwf4HrAVuMsZU2ChY3xM/wBNfb+LF2du4/9x23HNOW7fDOXXHDtnEXJRvT9lT1zkXxfLhaCrsXW67uRlja4a5GbaZSbzs9OoSEGbbmY0zNlJJbTN9z4nlBEXb5FvdWvSz3+2gM5xGeEuI7gg759kEGxZn90FIU6ebXrZN0C37233oF2QPPEFR9ixAvOw+jEuCkGbOBcoQe2DzD7ZnBUcP2IOq1qTVKaoo8df6JWwRaQ7cA3QyxuSIyIfANcBbtR1LbXrw/Pbsz8zlv9/+TEyoP1f3ael2SKcmqNQ9CZGtof2FJ1+nuMgmsoIc2yTk7QebvrRJu3Eru0xGsu0y5xvg3MHo1I4P77AX+IrybKIVL9snOmOPvSkmqp1Njoe3Q246dL7CJky/YFs7L2lbDgiD6A52mah29uzlyE4bW2w3G8uRnbYmnZlsDxj+obZpI323TbihzezyWSl2HbBJPDvNLufta88sxLv6as2l79QMa14921TK4VbfJR+gkYgUAIFAg38orYjwxJXdOHQ0n0c+XUd0iD9nd4hxO6ya5eUNXo1ObFLofNmJyzTv/ev1mvWo2bhiOp/4uaKxUEoOTiWCo395HxR14sFQL1CqeqTWG/WMMcnAU8BuIAXIMMZ8U9txuMHX24sXr+tFp9hQ7pi8gpW7j7gdklLKA9V64heRxsAoIAFoBgSJyG/KWe42EVkmIssOHqyBtlqXBPn78Ma4PnYwt7eWsvXAUbdDUkp5GDcu448AdhhjDhpjCoBPgIFlFzLGvGKMSTLGJEVHR/9qI/VZdIg/74zvi7eXcM0rC6v27F6llKombiT+3UB/EQkU26/xHGCjC3G4qlVkEFNvG4CPlxfXvrqIbQe15q+Uqh1utPEvBqYBK7BdOb2AV2o7jrqgTZNgPpjQH28v4YbXl7A/U5+5q5Sqea7csWGMecwY08EY08UYc70xJs+NOOqCVpFBvDmuL0ey8xn7yiJSMqowoqBSSp0BvVWvDugaF8Y74/tyICuPq15eyJ7D2W6HpJRqwDTx1xFJ8RFMvqUfmTmFXPXyQnYcOuZ2SEqpBkoTfx3SvUU479/an/zCYq56eSE/769oLHKllDp9mvjrmE7NQvlgQn8EuOaVRaxLznA7JKVUA6OJvw5q0ySEDycMoJGvN9e+ukjv8FVKVStN/HVUfFQQH0zoT3igH9e/voQlOw67HZJSqoHQxF+HxTUO5MMJA4gJ9efGN5Ywb8sht0NSSjUAmvjruKZhAXwwYQCtIgMZ//ZSfti03+2QlFL1nCb+eiAq2J+pt/WnQ9MQJry7nK/XpbgdklKqHtPEX0+EB/rx3i396BYXzp1TVvLKnG3Uh8dmKqXqHk389UhogC/vjO/L+Z1j+L+Zm/jzZ+spKtbkr5Q6NZr465kgfx+eH9uLCUMTeXfRLia+t5zs/EK3w1JK1SOa+OshLy/h4Qs78vglnfh+436ueWURB7J0ZE+lVNVo4q/Hxp2VwMvXJ7Fl/1Eun7SAzak6xINS6uQ08ddz53aK4cMJAygoKubKFxfw46YDboeklKrjNPE3AF3jwvjsrrOO9/V/dc527fGjlKqQJv4GIjasER9NHMCFXZryj5kbeWDaGvIKi9wOSylVB2nib0AC/WyPn3tHtGXa8r1c88oifZyjUupXNPE3MF5ewr0j2vHidb3YnJrFBc/M0Tt9lVIn0MTfQF3YNZYZd51Fy8ggbp+8Qu/0VUodp4m/AWvTJISpt/bngs5N+b+Zm7j9vRVk5ha4HZZSymWa+Bu4Rn7evHBdLx69uCPfbtzPpf+bx4Z9mW6HpZRykSZ+DyAi3DI4kam39Sc7v4jLXpjPlMW7telHKQ+lid+D9ImPYOZvB9MvIYJHPl3LPVNXkaVNP0p5HE38HiYq2J+3b+rLA+e358s1+7j0+fms36cPdFfKk2ji90BeXsKdw9vw/q39yc4v5PIXFvD2gp3a9KOUh9DE78H6JUYy857BnNU6ksdmrOfWd5Zz+Fi+22EppWqYJn4PFxnszxvj+vDnkZ2Y8/NBLnx2Dgu26UPdlWrIqpT4ReTdqkxT9ZOIMH5QAp/cMZAgPx+ue20x//lmM4VFxW6HppSqAVWt8Xcu/UFEvIHe1R+OclOX5mF8fprsoccAABluSURBVPcgxvSO438/bOWqlxey53C222EppapZpYlfRB4WkSygm4hkOq8s4ADwWa1EqGpVkL8PT47uznNje7Jl/1EuenYu01cmux2WUqoaVZr4jTH/NMaEAP82xoQ6rxBjTKQx5uHTLVREwkVkmohsEpGNIjLgdLelasal3Zsx87eDad80hHs/WMW9U1fqcA9KNRBVber5QkSCAETkNyLyXxFpdQblPgt8bYzpAHQHNp7BtlQNaRERyNTb+nPfiHZ8viaFC56ew4KteuFXqfquqon/RSBbRLoDvwO2Ae+cToEiEgYMAV4HMMbkG2PST2dbqub5eHvx2xFtmTZxAP6+3lz72mL++vkGcgv0IS9K1VdVTfyFxt7dMwp43hgzCQg5zTITgIPAmyKyUkReKzmbUHVXz5aN+fKeQVzfvxVvzN/BJf+bx7pkveNXqfqoqok/S0QeBq4HvhQRL8D3NMv0AXoBLxpjegLHgIfKLiQit4nIMhFZdvDgwdMsSlWnQD8f/nZZF94e35eMnAIumzSf53/Yot0+lapnqpr4rwbygPHGmFQgDvj3aZa5F9hrjFnsfJ6GPRCcwBjzijEmyRiTFB0dfZpFqZowtF00s+4dwvldmvLUNz8z5uWFbEzRoZ6Vqi+qlPidZD8ZCBORkUCuMea02vidbe0RkfbOpHOADaezLeWexkF+PD+2J89e04NdadmMmjSfD5bqUM9K1QdVvXP3KmAJMAa4ClgsIqPPoNy7gckisgboAfzfGWxLuUREGNWjOd/dP5R+CRH84eO1jJo0n11px9wOTSlVCalKDU1EVgPnGmMOOJ+jge+MMd1rOD4AkpKSzLJly2qjKHWaiooNHyzdw5OzNlFcbPjnFd24uFus22Ep5dFEZLkxJqns9Kq28XuVJH1H2imsqzyAt5dwbb+WfHbnWSREB3PnlBXc/8EqdhzS2r9SdU1Vk/fXIjJLRMaJyDjgS2BmzYWl6qtWkUFMmziAu4a3YcbqfZz39E9M+nGr9vxRqg6ptKlHRNoAMcaY+SJyBTDImZUOTDbGbKuFGLWpp546kJnLX77YwJdrUujeIpz/jOlOmybBboellMc43aaeZ4BMAGPMJ8aY+40x9wOfOvOUqlCT0AAmXduL/43tya60Y1z03FxenL1Na/9KuexkiT/GGLO27ERnWnyNRKQanEu6N+Obe4dwdvsmPPH1JkZNmq93/SrlopMl/vBK5jWqzkBUw9YkNICXru/Ni9f1Yn9mHqMmzeeJrzfpmD9KueBkiX+ZiNxadqKI3AIsr5mQVEN2YddYvr9/KFf2as6Ls7cx/KnZzNuiI34qVZtOdnE3Btuen88viT4J8AMud+7CrXF6cbdhWrQ9jT9NX8e2g0e5dUgiNw9KoElIgNthKdVgVHRxt6o3cA0Hujgf1xtjfqjm+Cqlib/hOpZXyKPT1zF9VTJNQvx5+uoeDGwd5XZYSjUIZ5T43aaJv+HbmJLJxPeWsystmyt7xfHHizsSEeTndlhK1WtneueuUjWqY2wos+4dwp3DW/PZqmTO+c9spizeTdrRPLdDU6rB0cSv6owAX28eOL8DM387mMToYB75dC2Dn/yRL9ekuB2aUg2KJn5V57SLCeGjCQOYNnEA7ZuGcOeUFfz9iw2kZ+e7HZpSDYImflUneXkJSfERfHDbAK7v34rX5u1g8BM/MnnxLu37r9QZ0sSv6jQ/Hy/+dlkXvrh7EG1jgvnjp+u4bNJ8Fm5Lczs0peotTfyqXujSPIyPbx/IazckcfhYPmNfXcSdU1aw7eBRt0NTqt7RxK/qDRFhRKcY5jw4nHvObsPsTQe48Jm5vLtwJ0XFdb9bslJ1hfbjV/XWwaw8fv/Ran76+SCJ0UHcNDCe0b1b0MjP2+3QlKoTtB+/anCiQ/x5Y1wfJl3bCy8R/vTZem6fvJyd+tQvpSqliV/Va95ewsXdYvn2viH8eWQnZm8+yHlPz+Hpb3/W7p9KVUATv2oQRITxgxKY/9DZDGsfzbPfb+Gc//zER8v26INflCpDE79qUJqHN+KVG5KYec9gWkQE8sC0NZzz35/4ck0KxXoBWClAE79qoDo1C+WT2wfy8vW9CfTz4c4pKxj61I+s3H3E7dCUcp0mftVgeXkJ53duyoy7zuLZa3pQVGS4/IUF3P7ecpLTc9wOTynX+LgdgFI1zdfbi1E9mjOsfRPemr+TF3/ayvcbD3BN3xb8aWQnfL21/qM8i/7FK48R1siX345oy3f3D+XK3nG8s3AXg574gTfn7yC/UC8AK8+hiV95nLjGgfzziq68MS6JxKhg/vL5Bs57+idmrN6nF4CVR9DErzzW2R1imHJrP968qQ9+Pl7c8/5KrnxpARtTMt0OTakapYlfeTQRYXj7Jnz92yH896ru7ErL5qLn5nLH5OX8vD/L7fCUqhF6cVcpbA+gK3rFMbx9E16ft4O3F+zk63WpXN2nJfed25YmIQFuh6hUtdFB2pQqx5Fj+Tz3wxbeXbgLfx8vJg5tzS2DE3UAOFWvVDRIm2uJX0S8gWVAsjFmZGXLauJXbtlx6BhPfr2Jr9alEh7oy5jecTxwfgf8fLSVVNV9dXF0zt8CG10sX6mTSogK4sXf9OajiQM4q00Ur87dQd//+453F+7ULqCq3nIl8YtIHHAx8Job5St1qvrERzDp2l68M74vnWJD+dNn6xn4L3sPgHYBVfWNWxd3nwEeBEJcKl+p0zKkXTSD2kQxZ8tBXp27nb98voFpy/cyslszJgxJxMtL3A5RqZOq9Rq/iIwEDhhjlp9kudtEZJmILDt48GAtRafUyXl5CcPaN+G9m/vx5Ohu5BYU8cTXm2j36Fe8MHsr9aHDhPJstX5xV0T+CVwPFAIBQCjwiTHmNxWtoxd3VV1mjGH6qmQ+WLqHRdsP0z8xghsHxDO0fTSBftpjWrmnzvXqARCRYcDvtVePagiKiw3vL93NE19tIjO3kI6xofzziq50jwtDRJuAVO2ri716lGpQvLyE6/q1Yv5DZ/PsNT1IycjhsknzSfr7d0xdstvt8JQ6Tm/gUqqGZGQXMGtDKtOW72XJjsNc1LUp941oR9sY7dOgakedbOqpKk38qj7LLyzm+R+38vrc7eQUFHFZj+Y8dFEHHQZC1ThN/Eq57PCxfF7+aRtvLdhJUbGhV6vG3DeiHf0TI/QagKoRmviVqiO27M9i8uLdfLEmhUNH8xjePpp7R7Sje4twt0NTDYwmfqXqmNyCIt5btIunv/2ZY/lF9GgRzriB8VzSvRneeiOYqgaa+JWqo7JyC/h4+V7eWbiL7YeO0T0ujPvObcdZbaL0ecDqjGjiV6qOKy42fL5mH3/7YiOHjubRpXkodw1vy/mdY/QagDotmviVqidy8ouYtT6VJ77eREpGLv0SIhg/KIHzOukBQJ0aTfxK1TNFxYY35+/g7YU72XM4hx4twrnprHgu6dZMB4NTVaKJX6l6qrComA+X7eXVudvZcegYnWJDeeCC9gxtG60HAFUpTfxK1XMl1wCe+mYzew7nkBAVxNi+Lbi8ZxzRIf5uh6fqIE38SjUQeYVFfL0ulRdnb2NTahahAT5MGNqaC7o0pXV0sNvhqTpEE79SDdDWA1n8+bP1LNiWhgjcfXZbbhuSSLC/DgetNPEr1aDtS8/h37M28+nKZBr5enNVUhy3DE6kRUSg26EpF2niV6qBM8awYFsan65M5rNVyRQVGy7sGsuEIYl0i9PhIDyRJn6lPEhqRi5vzt/BlMW7ycorpH9iBBOGtGZY+2i9F8CDaOJXygNl5Rbw/pLdvDFvJ6mZuXRvEc5fLu2sTwXzEJr4lfJg+YXFTF+VzN+/2EBmbiEtIhpx2+BExiS1IMDX2+3wVA3RxK+UIj07n2827OeDpXtYvusITUL8uW1IImP7tiRIewI1OJr4lVLHGWNYuC2N537YwqLthwkN8KF3q8Zc1rM5l3Zvps1ADYQmfqVUuVbsPsIb83awcnc6yek5DEiM5Np+Lbm4a6wOCVHPaeJXSlWqqNjwzsKdvDZ3B8npObSPCeG+c9tyfuemegZQT2niV0pVSXGx4cu1KTz93c9sP3iM+MhAzu4Qw/hB8cQ11hvC6hNN/EqpU1JYVMyM1fv4bNU+Fm5PwxjD0HbRXNuvJcPbN9GzgHpAE79S6rQlp+fw5rwdfLEmhdTMXHq0COd357VjUJsoPQDUYZr4lVJnrKComGnL9/K/77ewLyOXvgkRXNilKSM6xui4QHWQJn6lVLXJKyxi6pI9vDJnO8npOfh5ezG0fTTjz0pgQOtIt8NTDk38SqkasedwNn//cgPfbNiPMdC7VWNG945jWPtoYsMauR2eR9PEr5SqUWlH85j04za+2ZDK3iP2LOCqPnHcMawNzcL1AOAGTfxKqVpRVGzYfvAob8zfybTlewDonxjJJd2bMbx9E31MZC3SxK+UqnXJ6Tm8Nnc7szcfZMehYwT7+3D32W24oEtTWkUGuR1eg6eJXynlmuJiw5rkDP5v5kaW7DgMQPe4MB44vwMDW0fq0BA1pM4kfhFpAbwDxAAGeMUY82xl62jiV6rh2HM4m2837OflOdvYn5lHq8hA7jm7LcPaRxMZrM1A1akuJf5YINYYs0JEQoDlwGXGmA0VraOJX6mGJ7egiFnrU3lx9jY2pWbRyNebkd1iuWFAPF3jwtwOr0GoKPHX+gDcxpgUIMV5nyUiG4HmQIWJXynV8AT4ejOqR3Mu7hrL6r3pTF2yh6/WpfLR8r10bhbKxd1iuSqpBVF6FlDtXG3jF5F4YA7QxRiTWWbebcBtAC1btuy9a9euWo9PKVW7MnMLmLJ4N99v3M/SnUfw8/bi6j4tGNQ2inM6NMHH28vtEOuVOtPUc7xgkWDgJ+AfxphPKltWm3qU8jybU7N4YfZWZq1PJbegmMToIAa1iWJw22gGt43SR0ZWQZ1K/CLiC3wBzDLG/Pdky2viV8pzFRQVM3NtCpMX7Wb13nTyCotpEdGIi7rEcsewNoQF+rodYp1VZxK/2KH83gYOG2Purco6mviVUmCbgub+fIhX525n9d50BBjYOooxSXGc16kpjfz0LKC0upT4BwFzgbVAsTP5EWPMzIrW0cSvlCprXXIGX61L4bNV++wQET5e9E+MZOKQRAa2iXI7vDqhziT+06GJXylVkeJiw+Idh/lu435mrk0hJSOXDk1DGNg6iit6Nadzs1CPfWaAJn6lVIOXW1DEuwt38dPPB1my4zD5RcW0iwlmVI/mtI4OYkTHGI/qGaSJXynlUdKz8/lybQqfrEhm+a4jgB0m4o7hbUiICqJNdHCDHypCE79SymMdzMpj7paD/P3LjRw+lg9ATKg/Y/u2ZFCbKHq3atwgm4M08SulPN6xvEK2HjjK1gNH+XDZHhY7A8b1aBHOJd2bcWWv5oQH+rkcZfXRxK+UUmWkZ+czY/U+pizezabULAD6JUQwslssXZqH0T0uvF43B2niV0qpSizdeZgFW9P4ZOVedqVlA9AuJpjbhrTm7A5NiAiqf2cCmviVUqoKjDEkp+ewaPthXv5pG1sOHAXss4T7J0YwpncL4qPqx0NkNPErpdQpKi42rNxzhAVb0/h2437W78ukqNiQEBVEz5bhXNq9GR2ahtI0LMDtUMuliV8ppc7Q/sxcPl2ZzMrdR1i4LY3M3EIAOsWGMqJTDBd3jaV90xCXo/yFJn6llKpGeYVFzNtyiB2HjjF9VTLrkjPx9hLaNgkmrnEgVyXFMbxDE3xdvGFME79SStWgtKN5vD5vBxtTMlm/L5MDWXlEBPkxrF00CVFBdIkLY0jbaLxrsZdQnXkCl1JKNUSRwf48eEEHAAqLipm9+SDvL9nNnC0H+WRlMmBvGmsXE8Jv+rfi3I4xrnUV1Rq/UkrVsPTsfH7YdIDvNx1g7d4Mdh/OxsdLiI8KolfLcPrER3B5z+bVPo6QNvUopVQdUFBUzBdr9rFmbwYrd6ez53A2ac4wEh1jQxnZLZYBrSMJb+RLfGTQGZ0VaOJXSqk6yBjD9xsPsGzXEZbsSGPF7vTj8+IjA3n5+qTT7imkbfxKKVUHiQgjOsUwolMMAMnpOWxOzWR/Zh5frUulRUSjai9TE79SStUhzcMb0TzcJvuxfVvWSBme80QCpZRSgCZ+pZTyOJr4lVLKw2jiV0opD6OJXymlPIwmfqWU8jCa+JVSysNo4ldKKQ9TL4ZsEJGDwK7TXD0KOFSN4VQXjevU1NW4oO7GpnGdmoYYVytjTHTZifUi8Z8JEVlW3lgVbtO4Tk1djQvqbmwa16nxpLi0qUcppTyMJn6llPIwnpD4X3E7gApoXKemrsYFdTc2jevUeExcDb6NXyml1Ik8ocavlFKqFE38SinlYRp04heRC0Rks4hsFZGHXI5lp4isFZFVIrLMmRYhIt+KyBbnZ+NaiOMNETkgIutKTSs3DrGec/bfGhHpVctxPS4iyc4+WyUiF5Wa97AT12YROb8G42ohIj+KyAYRWS8iv3Wmu7rPKonL1X0mIgEiskREVjtx/cWZniAii53yPxARP2e6v/N5qzM/vpbjektEdpTaXz2c6bX2t++U5y0iK0XkC+dzze4vY0yDfAHewDYgEfADVgOdXIxnJxBVZtqTwEPO+4eAJ2ohjiFAL2DdyeIALgK+AgToDyyu5bgeB35fzrKdnN+nP5Dg/J69ayiuWKCX8z4E+Nkp39V9Vklcru4z53sHO+99gcXOfvgQuMaZ/hJwu/P+DuAl5/01wAc1tL8qiustYHQ5y9fa375T3v3AFOAL53ON7q+GXOPvC2w1xmw3xuQDU4FRLsdU1ijgbef928BlNV2gMWYOcLiKcYwC3jHWIiBcRGJrMa6KjAKmGmPyjDE7gK3Y33dNxJVijFnhvM8CNgLNcXmfVRJXRWplnznf+6jz0dd5GeBsYJozvez+KtmP04BzRERqMa6K1NrfvojEARcDrzmfhRreXw058TcH9pT6vJfK/zFqmgG+EZHlInKbMy3GGJPivE8FYtwJrcI46sI+vMs51X6jVFOYK3E5p9U9sbXFOrPPysQFLu8zp9liFXAA+BZ7dpFujCksp+zjcTnzM4DI2ojLGFOyv/7h7K+nRcS/bFzlxFzdngEeBIqdz5HU8P5qyIm/rhlkjOkFXAjcKSJDSs809tzN9b61dSUOx4tAa6AHkAL8x61ARCQY+Bi41xiTWXqem/usnLhc32fGmCJjTA8gDntW0aG2YyhP2bhEpAvwMDa+PkAE8IfajElERgIHjDHLa7Pchpz4k4EWpT7HOdNcYYxJdn4eAD7F/kPsLzl9dH4ecCm8iuJwdR8aY/Y7/6zFwKv80jRRq3GJiC82uU42xnziTHZ9n5UXV13ZZ04s6cCPwABsU4lPOWUfj8uZHwak1VJcFzhNZsYYkwe8Se3vr7OAS0VkJ7Y5+mzgWWp4fzXkxL8UaOtcHffDXgiZ4UYgIhIkIiEl74HzgHVOPDc6i90IfOZGfJXEMQO4wenh0B/IKNW8UePKtKlejt1nJXFd4/RwSADaAktqKAYBXgc2GmP+W2qWq/usorjc3mciEi0i4c77RsC52OsPPwKjncXK7q+S/Tga+ME5g6qNuDaVOngLth299P6q8d+jMeZhY0ycMSYem6N+MMZcR03vr+q8Ml3XXtgr8z9j2xj/6GIcidgeFauB9SWxYNvmvge2AN8BEbUQy/vYJoACbNvhzRXFge3RMMnZf2uBpFqO612n3DXOH3xsqeX/6MS1GbiwBuMahG3GWQOscl4Xub3PKonL1X0GdANWOuWvA/5c6n9gCfai8keAvzM9wPm81ZmfWMtx/eDsr3XAe/zS86fW/vZLxTiMX3r11Oj+0iEblFLKwzTkph6llFLl0MSvlFIeRhO/Ukp5GE38SinlYTTxK6WUh9HErxo8ETnq/IwXkWureduPlPm8oDq3r1RN0MSvPEk8cEqJv9TdkxU5IfEbYwaeYkxK1TpN/MqT/AsY7Iy7fp8zaNe/RWSpM0jXBAARGSYic0VkBrDBmTbdGWBvfckgeyLyL6CRs73JzrSSswtxtr1O7HMYri617dkiMk1ENonI5JLRFUXkX2LH118jIk/V+t5RHuNktRmlGpKHsGPVjwRwEniGMaaPMyrjfBH5xlm2F9DF2CGMAcYbYw47t/svFZGPjTEPichdxg78VdYV2IHSugNRzjpznHk9gc7APmA+cJaIbMQOsdDBGGNKhhdQqiZojV95svOw47Gswg5pHIkdwwZgSamkD3CPiKwGFmEHyWpL5QYB7xs7YNp+4CfsCJAl295r7EBqq7BNUBlALvC6iFwBZJ/xt1OqApr4lScT4G5jTA/nlWCMKanxHzu+kMgwYAQwwBjTHTvmS8AZlJtX6n0R4GPs2Op9sQ/XGAl8fQbbV6pSmviVJ8nCPqawxCzgdmd4Y0SknTN6allhwBFjTLaIdMA+iq9EQcn6ZcwFrnauI0RjHy1Z4WiYzrj6YcaYmcB92CYipWqEtvErT7IGKHKabN7CjnseD6xwLrAepPzHX34NTHTa4Tdjm3tKvAKsEZEVxg6nW+JT7Dj0q7GjaD5ojEl1DhzlCQE+E5EA7JnI/af3FZU6OR2dUymlPIw29SillIfRxK+UUh5GE79SSnkYTfxKKeVhNPErpZSH0cSvlFIeRhO/Ukp5mP8HXy6/BZfCOTAAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hXgiAe3LM8zo"
      },
      "source": [
        "## Training "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "79be990fb4d94d4eb00e7e40c34a7dc0",
            "d6bebec478ff445596d2d20e3e41d8d8",
            "7357c2dbe01b465a8025bb04079e1ba6",
            "789af892e98d4cd9945927f18d344b33",
            "5c87f37dea28475989e457a281b49bcb",
            "81ef136b29f448bbac4feb43dfe71d3b",
            "dec40c6f7fe943f28b70dcf421feeeaf",
            "1ec6627be3104bd1ac8f7527b49c612c",
            "103187e7a8c54bf287f1c280d87b24ab",
            "13862922e0b7405aa69aacb30866f962",
            "5c53dcfed97c4acc8bb0c68684c358db",
            "3c046fa78c6d4517aced2b9eac42a1b0",
            "1df95b56f524432c8de6767237c32a36",
            "35b71d1b0a124dbaad77f6a66dcf337a",
            "4008d352b6e84b908bb9e59fc7b07846",
            "9e06d9aa78374fb5b8e435a7926cd1fc",
            "08c8a169418142a28e90cc2cee731fd0",
            "37653c2234204b43afd44f9e409e18c3",
            "4076ba1f2023409c9f13da22659d4456",
            "96880343dafb49678199fe83d5dc4909",
            "595e5459dbd64350a1b1927da3895f99",
            "fce7fe73f9614d85a4cc788fe850de3d",
            "0dbeb0cf42604b9284961c661d7bc440",
            "73f2e6e91a1f461493a30bddb4dca257",
            "4b70b01b2f784ab7b4d03dc0690a8133",
            "11f7a56252c34ac9a610efa568f449c3",
            "c8432bf3291b41658bdea268bbe4a52c",
            "03e6704894e4460cbf9ac756b1b7fbf5",
            "553feaf967734e659b3e40e864890591",
            "9724c16f39804e65a2339f4eb0888d87",
            "35d8838d189b43b6a89e56456ce5cda6",
            "aec6f668a23f4ef3ab62edc211ee50e1",
            "a81de214bedf4591962bf94e5fbe6d05",
            "11ed2eec12394dcebbd6675ec09eae68",
            "44b106d095c249a182e4be352474017e",
            "ef737862bdc04cab8e983aae46c8d61a",
            "564e39703608415f877eb6ae697de7a0",
            "4a389d42259f4a0b91fef8be9bb3331b",
            "5ad8627d97bc484a8ad7e597bba19059",
            "96fd40b4f0f947cd86f8239cb0c05dfd",
            "3082e9e6a6ed41f0915fa2e83a517855",
            "216cc2fc3b52449081bd009710f6610a",
            "d8e3054ea7814cd49bc2c853ba00b438",
            "7378bead921449b197d2d07a69c9be89",
            "7fb8d548288c451981066b81db43907c",
            "776d8e3df4404e55b1940d4484be56a1",
            "9092ec82a52c4d7fbe5b23ea5b49929e",
            "e6608a381424445382708ea0f0ed2f30",
            "3b5227a489704d3c900eeb3a462bd282",
            "912aaef2f7d347d680a434c28f5094f9",
            "983239dac92b44a3bf23bef0a3b043f9",
            "ce572565bc8340589c4dccfc59499389",
            "e5ee78621e0743719c1f1affa1ef9342",
            "e2ab5b323e2f40779f69d3cbc9f48183",
            "01bd995c470b40b0b8044f1522b72ca3",
            "3f56b2be43c74873bd9e1f0fb7f0d5f1",
            "6bd9fbd4d1584092807987bf75cbd6b0",
            "50c88e7de0ac4237a27388f0e048d874",
            "8174f6f32337401f820325273ae84a5e",
            "0c87e696b3f54a1fad11004c923e9fd8",
            "53dc7fbe0da84bc6b988e05de9bc7ed1",
            "799a086984254677a0f75b888ac3e6d8",
            "20678248d9444021b140a2bab9ad6d06",
            "ca8e9854ae3e49b5a66ecd87e04de47e",
            "adcaf598152841c89569fdb0831f46c4",
            "fb0d56e5b3484062bb2d7956750933ef",
            "52200d85df18452a912040057168d8b1",
            "831e5d06718c457abbd10ce957debeb5",
            "270c66b7748e411e86beab3b320af455",
            "b57d3a2256da42e9a82b7cb82245a307",
            "1e9a54e5fca646c9a7945eac9838581c",
            "60172a871b024528a3ed118626522b8c",
            "a32d997d0cec451ab443f0d1e2dacb5c",
            "d549d1053f6b4cd3a9c98fe80208fccb",
            "332395b1ef6641f5bcbfa133f88b1375",
            "64599e3a0b80469fbf5bf0fb97655da4",
            "2a5f82b38d4348ac9dfc8a33d2beb500",
            "7040547c40804dcd9020166408347054",
            "cbac7175a6384253b163299ad469966e",
            "31befd519b2744dc9139d685090bb453"
          ]
        },
        "id": "dWbyIG5UM_Lq",
        "outputId": "add680ae-51bd-4f94-a2aa-748fedebe72a"
      },
      "source": [
        "torch.manual_seed(2045)\n",
        "\n",
        "pad_idx = 0\n",
        "\n",
        "# Parameters\n",
        "params = {'batch_size': 128,\n",
        "          'shuffle': True,\n",
        "          'num_workers':2,\n",
        "          'collate_fn' : BatchPadCollate(pad_idx=pad_idx)\n",
        "          }\n",
        "\n",
        "# Training and validation Data Generators\n",
        "training_set = AFEC_Dataset(en_train_dic,fa_train_dic,partition['train'],en_vocab_dic, fa_vocab_dic)\n",
        "training_generator = torch.utils.data.DataLoader(training_set, **params)\n",
        "\n",
        "validation_set =  AFEC_Dataset(en_train_dic,fa_train_dic,partition['validation'],en_vocab_dic, fa_vocab_dic)\n",
        "validation_generator = torch.utils.data.DataLoader(validation_set, **params)\n",
        "\n",
        "# Hyperparameters\n",
        "embed_dim = 256\n",
        "num_heads = 8\n",
        "dim_key = 64\n",
        "dim_value = 64\n",
        "dim_inner = 1024\n",
        "num_layers = 3\n",
        "src_vocab_size = len(en_vocab_dic.get_str_2_idx())\n",
        "trg_vocab_size = len(fa_vocab_dic.get_str_2_idx())\n",
        "learning_rate = 1e-4\n",
        "num_iterations = 5000\n",
        "\n",
        "kwargs = {'p_dropout': 0.1,\n",
        "          'kdim': dim_key,\n",
        "          'vdim':dim_value,\n",
        "          }\n",
        "# Model\n",
        "# model = Transformer(num_layers, embed_dim, dim_inner, num_heads, src_vocab_size,trg_vocab_size, device,**kwargs).to(device)\n",
        "\n",
        "# Define Loss function and Optimizer\n",
        "criterion = nn.CrossEntropyLoss(ignore_index = pad_idx)\n",
        "optimizer = optim.Adam(model.parameters(), lr = learning_rate, betas=(0.9, 0.98), eps=1e-9)\n",
        "\n",
        "# train_losses = []\n",
        "# validation_losses = []\n",
        "\n",
        "torch.autograd.set_detect_anomaly(True)\n",
        "\n",
        "train_dataiter = iter(training_generator)\n",
        "\n",
        "trainloss, valloss, running_trainloss, running_valloss = 0.0, 0.0, 0.0, 0.0\n",
        "\n",
        "# Loop over epochs\n",
        "for itr in tqdm(range(num_iterations)):\n",
        "\n",
        "    # Training        \n",
        "    batch_train_src, batch_train_trg = train_dataiter.next()\n",
        "\n",
        "    # Transfer to GPU\n",
        "    batch_train_src, batch_train_trg = batch_train_src.to(device), batch_train_trg.to(device)\n",
        "\n",
        "    # zero the parameter gradients\n",
        "    optimizer.zero_grad()\n",
        "    \n",
        "    # Model computations\n",
        "    # forward + backward + optimize\n",
        "    batch_train_outputs = model(batch_train_src, batch_train_trg[: , :-1])\n",
        "    \n",
        "    loss = criterion(torch.transpose(batch_train_outputs,1,2), batch_train_trg[: , 1:])\n",
        "\n",
        "    loss.backward()\n",
        "\n",
        "    # Clip to avoid exploding gradient issues, makes sure grads are within a healthy range\n",
        "    nn.utils.clip_grad_norm_(model.parameters(), max_norm=1,error_if_nonfinite= False)\n",
        "\n",
        "    optimizer.step()\n",
        "\n",
        "    # print statistics\n",
        "    trainloss += loss.item()\n",
        "    \n",
        "    # Validation\n",
        "    if itr % 500 == 499:\n",
        "        with torch.set_grad_enabled(False):\n",
        "            val_cnt = 1\n",
        "            for batch_src, batch_trg in tqdm(validation_generator):\n",
        "                # Transfer to GPU\n",
        "                batch_src, batch_trg = batch_src.to(device), batch_trg.to(device)\n",
        "                \n",
        "                # Model computations\n",
        "                # forward + backward + optimize\n",
        "                batch_outputs = model(batch_src, batch_trg[: , :-1])\n",
        "                \n",
        "                val_loss = criterion(torch.transpose(batch_outputs,1,2), batch_trg[: , 1:])\n",
        "\n",
        "                valloss += val_loss.item()\n",
        "                val_cnt += 1\n",
        "\n",
        "        print(\"Cost of Train data after %i iterations : %f\" %(itr + 1, trainloss / 500))\n",
        "        print(\"Cost of Validation data after %i iterations: %f\" %(itr + 1, valloss / val_cnt))\n",
        "        running_valloss = 0\n",
        "\n",
        "        train_losses.append(trainloss/500)\n",
        "        trainloss = 0\n",
        "        validation_losses.append(valloss/val_cnt)\n",
        "        valloss = 0 "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "79be990fb4d94d4eb00e7e40c34a7dc0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=5000.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "103187e7a8c54bf287f1c280d87b24ab",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 500 iterations : 4.077437\n",
            "Cost of Validation data after 500 iterations: 4.177563\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "08c8a169418142a28e90cc2cee731fd0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 1000 iterations : 4.076545\n",
            "Cost of Validation data after 1000 iterations: 4.164702\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4b70b01b2f784ab7b4d03dc0690a8133",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 1500 iterations : 4.067843\n",
            "Cost of Validation data after 1500 iterations: 4.154782\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a81de214bedf4591962bf94e5fbe6d05",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 2000 iterations : 4.059635\n",
            "Cost of Validation data after 2000 iterations: 4.142466\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3082e9e6a6ed41f0915fa2e83a517855",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 2500 iterations : 4.053846\n",
            "Cost of Validation data after 2500 iterations: 4.135002\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3b5227a489704d3c900eeb3a462bd282",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 3000 iterations : 4.037088\n",
            "Cost of Validation data after 3000 iterations: 4.124817\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6bd9fbd4d1584092807987bf75cbd6b0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 3500 iterations : 4.040361\n",
            "Cost of Validation data after 3500 iterations: 4.117778\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "adcaf598152841c89569fdb0831f46c4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 4000 iterations : 4.026344\n",
            "Cost of Validation data after 4000 iterations: 4.109282\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a32d997d0cec451ab443f0d1e2dacb5c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 4500 iterations : 4.020610\n",
            "Cost of Validation data after 4500 iterations: 4.098103\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "StopIteration",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mStopIteration\u001b[0m                             Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-41-ddd6732e63ee>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m     \u001b[0;31m# Training\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 55\u001b[0;31m     \u001b[0mbatch_train_src\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_train_trg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_dataiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     56\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m     \u001b[0;31m# Transfer to GPU\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    519\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sampler_iter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    520\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 521\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    522\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    523\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1174\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_persistent_workers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1175\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_shutdown_workers\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1176\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1177\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1178\u001b[0m             \u001b[0;31m# Now `self._rcvd_idx` is the batch index we want to fetch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mStopIteration\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "1d229d2b4ae147f0a07842bd01eb68ce",
            "20b7c7394f8b4cefa1439d0c12cd649f",
            "88bb4390fdbc49258913f6da7ad74205",
            "766075418f0f49049d1c7aeafb4a4b25",
            "92e8c6e1fbcf4c9296c373107d4eaac4",
            "d3544e728d73447d88ff4ef0caff2bd4",
            "b70ed59302a34f2bb991aebbc7fd0ddf",
            "e54ed8ec9f56400c8f19bbdf4d038a3d",
            "0605a2f5fd8c4b62b69edb3fb23015df",
            "f20845546eda4539bfe972dd55f18717",
            "6323cdf06f96463dae17aaa9c4eb9ec9",
            "8d8f0ec30bc3443cbc685259f2c83629",
            "66eadeb29e2e48b8aa237448e2703512",
            "e561633365a84a5dbd83b1e8b41bb793",
            "df863cbfda3040d7a7867ab96a3ef478",
            "059cc0a36426450381df6122f294b7c9",
            "f7a6ca09cb2b4c7797ba0f98430a6866",
            "6409278f894a47ecb484e0d1fb94fdce",
            "acf4771e6efd41a68e8a2c42872cca8d",
            "2f61fad46b174b28b945520232a6bf6b",
            "9cff60fce822429eb1a52dea6f6aec32",
            "9df6adfb8899408483b095c835e58955",
            "b4fa9dd936a647a496d0c94e594cc70a",
            "6a2b8bdb81e742df9ad55a91146eb4ef",
            "aa026421293549afa11196fa0d06e2a3",
            "3e87d692436a47fd898215c8f82ba3bd",
            "8bdb740d9ba64b778854c4bdb15bddfc",
            "4ee532e0a6224b92a26852dd47f1e2de",
            "445276748bf44d12807f5a29afe14c85",
            "b0c3a7176d164f0c913311981a28785c",
            "d71508d2848d4733bca3cb3d2dd821f9",
            "948f29f553bd4b44bd5b0f82076b281d",
            "d34e99f0ea4144e6a7a94c030030c59f",
            "ee3c9df8970946088c31359fd5ded64e",
            "68f0dde2f22d4fe7bc65cf0be8f18bd2",
            "61006b21dd264cf2b6ef90430724ae23",
            "68634acf1af4485c932a1ddaabb92467",
            "a758531ed2584c03966021adf0ad6261",
            "82c510a02db4432696e36a13de338b03",
            "f0ce220e9b8d4329811771e11d1b4dac",
            "16b1ae35ffd74bc98ef4bc7523524aed",
            "c8045c741b01455aaa9371b3ba07e50f",
            "22afe2b68d65415fb46c827c7dd97809",
            "87e337067d814094967507acd350be80",
            "a6fe6eaf5474437486632b51388d3755",
            "a57d7d3a92b848058776dce01c6c6e8e",
            "22048c9838c74bc593da208e812edef0",
            "66831f3c2b9440c98107c162066603d5",
            "afc4b3f7af6f48238794450d90662082",
            "08b26a14bdf54ea0899e236eb15a9699",
            "7366d75e65c04958a9d6195b14042291",
            "e0071200473d49058ba25589a9aed2c0",
            "43939e07e3144b97b9a23e5b5a7396e6",
            "dd4e16a8091b437d90f93e2d85526f69",
            "e21c1e1230764cbca572a7cec1f9ce2c",
            "530c2e0b7dab4e42ab5646f9e6b88f03",
            "1f5853e0803d42298a4481dbee217c7f",
            "cd827ecc14be43acb2201773824931e8",
            "997eb1ceb61a4278846212428f890ad5",
            "636a9792555a40438c5e367ad20cdddf",
            "cb81f1cb82484d2a85aeaef84498adf4",
            "9c96446d5e6944ba8897cd734b4ca20c",
            "fde49670538348908feae72d735010dd",
            "92d39007791145f8aecfb3a4a556a196",
            "c350b3cc6be44435ab9513ffbfdd534f",
            "fa28262d042f44448a657e4b85b4a4f8",
            "01032022e4ae4589a4a922ed3c56ae53",
            "06fe92a7afd54d9f8354d94defc2f355",
            "17e5e2f0764343588b4b05a57812aed1",
            "6f4b9aece3054fa7bae7a922d506aca2",
            "afc7292e11784eda93c173d7300b1992",
            "b2b9dd12d9484553a52aa5e25be363a0",
            "856c7f0665dd4b1a8e20c63287455482",
            "d8392e8dba6d4518964cb208da74e97d",
            "cfb807635c2e410dade5ac5b91288bda",
            "f58c295313c94346b663706bc2ad23a4",
            "dd916a24ebfe4358b8f3b31dfe339af9",
            "3cb99ab7eb0a44d8b5da66eb5bd2e993",
            "37659054e0c94312900d986de97f1360",
            "e147b70b74e740bdaf7e3234df35368b"
          ]
        },
        "id": "RGEXEDZNpo8R",
        "outputId": "b4ad24bf-0325-4fe0-e139-62cf8c17b59a"
      },
      "source": [
        "train_dataiter = iter(training_generator)\n",
        "trainloss, valloss, running_trainloss, running_valloss = 0.0, 0.0, 0.0, 0.0\n",
        "\n",
        "num_iterations = 5000\n",
        "\n",
        "# Loop over epochs\n",
        "for itr in tqdm(range(num_iterations)):\n",
        "\n",
        "    # Training        \n",
        "    batch_train_src, batch_train_trg = train_dataiter.next()\n",
        "\n",
        "    # Transfer to GPU\n",
        "    batch_train_src, batch_train_trg = batch_train_src.to(device), batch_train_trg.to(device)\n",
        "\n",
        "    # zero the parameter gradients\n",
        "    optimizer.zero_grad()\n",
        "    \n",
        "    # Model computations\n",
        "    # forward + backward + optimize\n",
        "    batch_train_outputs = model(batch_train_src, batch_train_trg[: , :-1])\n",
        "    \n",
        "    loss = criterion(torch.transpose(batch_train_outputs,1,2), batch_train_trg[: , 1:])\n",
        "\n",
        "    loss.backward()\n",
        "\n",
        "    # Clip to avoid exploding gradient issues, makes sure grads are within a healthy range\n",
        "    nn.utils.clip_grad_norm_(model.parameters(), max_norm=2,error_if_nonfinite= False)\n",
        "\n",
        "    optimizer.step()\n",
        "\n",
        "    # print statistics\n",
        "    trainloss += loss.item()\n",
        "    \n",
        "    # Validation\n",
        "    if itr % 500 == 499:\n",
        "        with torch.set_grad_enabled(False):\n",
        "            val_cnt = 1\n",
        "            for batch_src, batch_trg in tqdm(validation_generator):\n",
        "                # Transfer to GPU\n",
        "                batch_src, batch_trg = batch_src.to(device), batch_trg.to(device)\n",
        "                \n",
        "                # Model computations\n",
        "                # forward + backward + optimize\n",
        "                batch_outputs = model(batch_src, batch_trg[: , :-1])\n",
        "                \n",
        "                val_loss = criterion(torch.transpose(batch_outputs,1,2), batch_trg[: , 1:])\n",
        "\n",
        "                valloss += val_loss.item()\n",
        "                val_cnt += 1\n",
        "\n",
        "        print(\"Cost of Train data after %i iterations : %f\" %(itr + 1, trainloss / 500))\n",
        "        print(\"Cost of Validation data after %i iterations: %f\" %(itr + 1, valloss / val_cnt))\n",
        "        running_valloss = 0\n",
        "\n",
        "        train_losses.append(trainloss/500)\n",
        "        trainloss = 0\n",
        "        validation_losses.append(valloss/val_cnt)\n",
        "        valloss = 0 "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1d229d2b4ae147f0a07842bd01eb68ce",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=5000.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0605a2f5fd8c4b62b69edb3fb23015df",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 500 iterations : 4.214147\n",
            "Cost of Validation data after 500 iterations: 4.273641\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f7a6ca09cb2b4c7797ba0f98430a6866",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 1000 iterations : 4.207669\n",
            "Cost of Validation data after 1000 iterations: 4.259844\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "aa026421293549afa11196fa0d06e2a3",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 1500 iterations : 4.193605\n",
            "Cost of Validation data after 1500 iterations: 4.246597\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d34e99f0ea4144e6a7a94c030030c59f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 2000 iterations : 4.182596\n",
            "Cost of Validation data after 2000 iterations: 4.232274\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "16b1ae35ffd74bc98ef4bc7523524aed",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 2500 iterations : 4.181644\n",
            "Cost of Validation data after 2500 iterations: 4.219688\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "afc4b3f7af6f48238794450d90662082",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 3000 iterations : 4.161390\n",
            "Cost of Validation data after 3000 iterations: 4.214949\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1f5853e0803d42298a4481dbee217c7f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 3500 iterations : 4.161115\n",
            "Cost of Validation data after 3500 iterations: 4.196480\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c350b3cc6be44435ab9513ffbfdd534f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 4000 iterations : 4.144227\n",
            "Cost of Validation data after 4000 iterations: 4.187565\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "856c7f0665dd4b1a8e20c63287455482",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 4500 iterations : 4.141772\n",
            "Cost of Validation data after 4500 iterations: 4.180213\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "StopIteration",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mStopIteration\u001b[0m                             Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-48-391bfceefbcd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0;31m# Training\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m     \u001b[0mbatch_train_src\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_train_trg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_dataiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0;31m# Transfer to GPU\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    519\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sampler_iter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    520\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 521\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    522\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    523\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1174\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_persistent_workers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1175\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_shutdown_workers\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1176\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1177\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1178\u001b[0m             \u001b[0;31m# Now `self._rcvd_idx` is the batch index we want to fetch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mStopIteration\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "_RStvVfbIfXS",
        "outputId": "2ea62cfd-c791-4ff4-9365-7443146b5583"
      },
      "source": [
        "# plot the cost\n",
        "plt.plot(range(500,27500,500),train_losses)\n",
        "plt.plot(range(500,27500,500),validation_losses)\n",
        "plt.title(\"Cost of Transfromer vs iters \")\n",
        "plt.ylabel('Cost')\n",
        "plt.xlabel('Iterations')\n",
        "plt.legend(labels = [\"Train\",\"Validation\"])\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU5dXA8d/Jvu9hDRB2FNmDiiiLWlsUxQ3Xqkgr1bZ2se5t1ddqtXV53apWa/V1RetC3deCUhcgICD7GiAJkJCE7OvkvH/cGxhiEgJkMknmfD+f+5k7d5vzzCRz5nmee58rqooxxpjAFeTvAIwxxviXJQJjjAlwlgiMMSbAWSIwxpgAZ4nAGGMCnCUCY4wJcJYITIcjIhNFZKOIlInI2f6OpzER6S4iX4hIqYg84O942oOI9HU/j2B/x2LaniWCACcil4hIpvtPvlNEPhCRE4/wmFkicuoRHOJO4DFVjVHVeY2OXeY11YtIpdfzS48k7kMwB9gDxKnq79rpNf1KVbe7n4cHQEQWiMhP/R2XaRuWCAKYiFwHPAT8GegO9AUeB2b4My6gH7C6qRXul1GMqsYA24EzvZa91LCdiIT4OL412szVmD5+7Rb587UPhdUsOhhVtSkAJyAeKANmtrBNOE6iyHWnh4Bwd10K8C6wFygEFuL8sHgBqAcq3ePf2MyxrwI2ufu+DfRyl29utH94C/FlAae681OAbOAmYJcbR6IbYz5Q5M6nee2/APgT8CVQCnwMpLjrIoAXgQK3jEtwkuVzQC1Q48Z3KnAH8Lq7fQnwU6CXW65Ct5xXeb3uHcC/3O1Lge+AIcAtQB6wAzit0Wf1DLATyAHuAoLddbPc+P/XjfWuRu9RL/e9TPJaNganRhMKDAI+B4rdZa82816nAwqEAHcDHqDKfQ8ec7cZBnzilnk9cIHX/s8BTwDvA+Xu+3Y6sMZ9D3KA6/39fxGok98DsMlPHzz8CKgDQlrY5k7gG6AbkAp8BfzJXXcP8KT7ZRIKnASIuy4L9wu6meOe7H7pjMVJNo8CX3itb3H/prbDSQR1wF/cY0YCycB5QBQQ6375zvPafwFO4hnibr8AuNdd9zPgHXffYGAcTlNQw5faXV7HuQMnOZyNkwwjgS9walcRwGicZHSy1/ZVwA/dL9bnga3A79338ipgq9fx3wL+DkS7n8Vi4Gfuulluua91jxXZxPv0Hw5MRPcBT7rzr7ivG+TGemIz73U6biLweu9+6rU+GieBXenG0ZBsjvZ6z4qBiV6vtRM4yV2fCIz19/9FoE7WNBS4koE9qlrXwjaXAneqap6q5gP/A1zmrqsFegL9VLVWVReq+x/dCpcC/1TVZapajfNLeIKIpB9OQbzUA7erarWqVqpqgaq+oaoVqlqK80t2cqN9nlXVDapaCbyG86XdUL5kYJCqelR1qaqWtPDaX6vqPFWtx6ktTQRuUtUqVV0O/AO43Gv7har6kfv+/wsn0d6rqrXAXCBdRBJEpDvOL+ffqGq5qubh/Pq/yOtYuar6qKrWueVo7GXgYgAREXffl73K2Q+nRlalqv9toYwtmQ5kqeqzbhzfAm8AM722+beqfqmq9apa5b720SISp6pFqrrsMF/bHCFLBIGrAEg5SJtyL2Cb1/Nt7jJwflVuAj4WkS0icvMhvPYBx1XVMjee3odwjKbku18wAIhIlIj8XUS2iUgJzq/0hEbt07u85iuAGHf+BeAjYK6I5IrIX0UktIXX3uE13wsodJNPg20cWL7dXvOVOEnZ4/UcN5Z+OLWEnSKyV0T24tQOujXz2k15AyfR9gQm4STMhe66GwEBFovIahGZfZBjNacfcFxDjG6clwI9WojzPJwkt01EPheRCYf52uYIWSIIXF8D1TjNGc3JxfkHb9DXXYaqlqrq71R1AHAWcJ2InOJud7CawQHHFZFonF/fOYdUgu9r/Lq/A4YCx6lqHM6XIDhffC0fyKnl/I+qHg2cgPOL9/KWdvGazwWSRCTWa1lfDq98O3A+pxRVTXCnOFUd3sxrfz8w1SKc/o8LgUuAuQ21N1XdpapXqWovnOawx0VkUCviavyaO4DPvWJMUKcD/5rm9lHVJao6AyepzcOpkRk/sEQQoFS1GLgN+JuInO3+eg4VkWki8ld3s1eAP4hIqoikuNu/CCAi00VkkNvUUIzTeVjv7rcbGNDCy78CXCkio0UkHOespUWqmtXGxYzF+XW9V0SSgNtbu6OITBWREW7toQSnGaP+ILsBoKo7cPpT7hGRCBEZCfwE9707FKq6E+dL/AERiRORIBEZKCKNm7gO5mWcRHY++5uFEJGZIpLmPi3C+bJuTTkbf8bvAkNE5DL37yhURMaLyFFN7SwiYSJyqYjEu81hJa18XeMDlggCmKo+AFwH/AGnM3MH8EucX2fgnJ2SCazEObNlmbsMYDDwKc5ZI18Dj6vqfHfdPTgJZK+IXN/E634K/BGnyWInMJAD27zbykM4Hbd7cDq9PzyEfXvgnAlUAqzFObPmhUPY/2KcDtZcnM7e291yH47LgTCcM2yK3Lh6HuIx3sb5zHap6gqv5eOBRSJS5m7za1Xd0orjPQycLyJFIvKI2wx2Gs7nmIvT5NbQcd+cy4Ast9nuapymJOMHDWd5GGOMCVBWIzDGmABnicAYYwKcJQJjjAlwlgiMMSbAdYoBqrylpKRoenq6v8MwxphOZenSpXtUNbWpdZ0uEaSnp5OZmenvMIwxplMRkW3NrbOmIWOMCXCWCIwxJsBZIjDGmADX6foIjDFdR21tLdnZ2VRVVR18Y9MqERERpKWlERra0mC5B7JEYIzxm+zsbGJjY0lPT8cZv9AcCVWloKCA7Oxs+vfv3+r9rGnIGOM3VVVVJCcnWxJoIyJCcnLyIdewLBEYY/zKkkDbOpz3M2ASwbpdJfz1w3UUV9T6OxRjjOlQfJYIRGSoiCz3mkpE5DeNthEReURENonIShEZ66t4thdU8PiCzWwvrPDVSxhjOpmCggJGjx7N6NGj6dGjB7179973vKampsV9MzMz+dWvftVOkfqWzzqLVXU97o3A3bs85eDcoMPbNJybZQwGjgOecB/bXK+ESABy9lYyIi3eFy9hjOlkkpOTWb58OQB33HEHMTExXH/9/nsp1dXVERLS9NdkRkYGGRkZ7RKnr7VX09ApwGZVbXyJ8wzgeXV8g3Nj8UO981Kr9PZKBMYY05xZs2Zx9dVXc9xxx3HjjTeyePFiJkyYwJgxYzjhhBNYv349AAsWLGD69OmAk0Rmz57NlClTGDBgAI888og/i3DI2uv00Ytw7lPbWG+c2yM2yHaX7fTeSETmAHMA+vbte1gBJESFEhkaTK4lAmM6pP95ZzVrckva9JhH94rj9jOHH/J+2dnZfPXVVwQHB1NSUsLChQsJCQnh008/5dZbb+WNN9743j7r1q1j/vz5lJaWMnToUK655ppDOpffn3yeCEQkDDgLuOVwj6GqTwFPAWRkZBzWvTVFhF4JEZYIjDEHNXPmTIKDgwEoLi7miiuuYOPGjYgItbVNn3ByxhlnEB4eTnh4ON26dWP37t2kpaW1Z9iHrT1qBNOAZaq6u4l1OUAfr+dp7jKf6J0YZYnAmA7qcH65+0p0dPS++T/+8Y9MnTqVt956i6ysLKZMmdLkPuHh4fvmg4ODqaur83WYbaY9+ggupulmIYC3gcvds4eOB4pVdWcz2x6x3gkR5Oy1S9mNMa1XXFxM7969AXjuuef8G4yP+DQRiEg08APgTa9lV4vI1e7T94EtwCbgaeDnvoynV3wke8qqqar1+PJljDFdyI033sgtt9zCmDFjOtWv/EMhqofV5O43GRkZerg3pnljaTa/+9cKFlw/hfSU6IPvYIzxqbVr13LUUUf5O4wup6n3VUSWqmqT57sGzJXFsP9aAusnMMaY/QIqETRcS5BticAYY/YJqETQPT4cEasRGGOMt4BKBOEhwaTGhFsiMMYYLwGVCMDpJ8i1U0iNMWafgEsEvRMjrUZgjDFeAi8RJESSs7eSznbarDGm7U2dOpWPPvrogGUPPfQQ11xzTZPbT5kyhYbT108//XT27t37vW3uuOMO7r///hZfd968eaxZs2bf89tuu41PP/30UMNvMwGXCHrFR1BdV09hectjjRtjur6LL76YuXPnHrBs7ty5XHzxxQfd9/333ychIeGwXrdxIrjzzjs59dRTD+tYbSHwEsG+awmsn8CYQHf++efz3nvv7bsJTVZWFrm5ubzyyitkZGQwfPhwbr/99ib3TU9PZ8+ePQDcfffdDBkyhBNPPHHfMNUATz/9NOPHj2fUqFGcd955VFRU8NVXX/H2229zww03MHr0aDZv3sysWbN4/fXXAfjss88YM2YMI0aMYPbs2VRXV+97vdtvv52xY8cyYsQI1q1b12bvQ3sNQ91h2A1qjOmgPrgZdn3XtsfsMQKm3dvs6qSkJI499lg++OADZsyYwdy5c7ngggu49dZbSUpKwuPxcMopp7By5UpGjhzZ5DGWLl3K3LlzWb58OXV1dYwdO5Zx48YBcO6553LVVVcB8Ic//IFnnnmGa6+9lrPOOovp06dz/vnnH3CsqqoqZs2axWeffcaQIUO4/PLLeeKJJ/jNb5ybO6akpLBs2TIef/xx7r//fv7xj3+0xbsUeDUCu0GNMcabd/NQQ7PQa6+9xtixYxkzZgyrV68+oBmnsYULF3LOOecQFRVFXFwcZ5111r51q1at4qSTTmLEiBG89NJLrF69usVY1q9fT//+/RkyZAgAV1xxBV988cW+9eeeey4A48aNIysr63CL/D0BVyOwG9QY00G18Mvdl2bMmMFvf/tbli1bRkVFBUlJSdx///0sWbKExMREZs2aRVXV4TUlz5o1i3nz5jFq1Ciee+45FixYcESxNgx13dbDXAdcjcBuUGOM8RYTE8PUqVOZPXs2F198MSUlJURHRxMfH8/u3bv54IMPWtx/0qRJzJs3j8rKSkpLS3nnnXf2rSstLaVnz57U1tby0ksv7VseGxtLaWnp9441dOhQsrKy2LRpEwAvvPACkydPbqOSNi/gEgHYDWqMMQe6+OKLWbFiBRdffDGjRo1izJgxDBs2jEsuuYSJEye2uO/YsWO58MILGTVqFNOmTWP8+PH71v3pT3/iuOOOY+LEiQwbNmzf8osuuoj77ruPMWPGsHnz5n3LIyIiePbZZ5k5cyYjRowgKCiIq6++Gl8LqGGoG9zy5ko+WZNH5h/8d7qWMcaGofYVG4a6FewGNcYYs19gJgL3zKFdxXYtgTHGBHQisH4CY/yvszVPd3SH834GZCKwawmM6RgiIiIoKCiwZNBGVJWCggIiIiIOab+Au44A9t+gxhKBMf6VlpZGdnY2+fn5/g6ly4iIiCAtLe2Q9gnIRGA3qDGmYwgNDaV///7+DiPgBWTTENgNaowxpkHAJgK7QY0xxjgCNxHYDWqMMQbwcSIQkQQReV1E1onIWhGZ0Gj9FBEpFpHl7nSbL+PxZjeoMcYYh687ix8GPlTV80UkDIhqYpuFqjrdx3F8j/cNapJjwtv75Y0xpsPwWY1AROKBScAzAKpao6rfv8Gnn/SyawmMMQbwbdNQfyAfeFZEvhWRf4hIdBPbTRCRFSLygYgMb+pAIjJHRDJFJPOwzzfOWwef3A61zhd/b7u62BhjAN8mghBgLPCEqo4ByoGbG22zDOinqqOAR4F5TR1IVZ9S1QxVzUhNTT28aIq2wpcPQc5SYP8NaqxGYIwJdL5MBNlAtqoucp+/jpMY9lHVElUtc+ffB0JFJMUn0fQ9HhDY9hVgN6gxxpgGPksEqroL2CEiQ91FpwAH3PhTRHqIiLjzx7rxFPgkoMhE6D4ctn25b5HdoMYYY3x/1tC1wEvuGUNbgCtF5GoAVX0SOB+4RkTqgErgIvXlif39ToBvXwRPLQSH0jshgjW5JT57OWOM6Qx8mghUdTnQ+I44T3qtfwx4zJcxHKDfCbD4Kdi5AtIyDrhBTURocLuFYYwxHUlgXVnc9wTn0W0eshvUGGNMoCWC2O6QPGhfh7HdoMYYYwItEYDTPLTta6j32A1qjDGGgEwEE6G6GPLW2A1qjDGGgEwEDf0EX9sNaowxhkBMBAl9Ib7PAR3GdoMaY0wgC7xEAG4/wVegSu/ESGsaMsYEtMBNBOV5ULCZId1iySoot/sSGGMCVoAmgonO47YvmTI0FVVYuPEwRzU1xphOLjATQfIgiE6FbV8xonc8ydFhzF+X5++ojDHGLwIzEYjs6ycIChImD0nl8w35eOrt/sXGmMATmIkAnOah4u2wdztThnWjqKKWldkd5gZqxhjTbgI4Eey/nmDS4BSCBOavt34CY0zgCdxE0O1oiIiH7V+REBXGmL6JLFhv/QTGmMATuIkgKBj6Ttg3AN3UoamszC4mv7Taz4EZY0z7CtxEAE7z0J4NUJbPlKHdAPhigzUPGWMCS4AnAvd6gu1fMbxXHKmx4cy35iFjTIAJ7ETQcxSERsG2rxARpgxJ5YsN+dR56v0dmTHGtJvATgTBodDnWMhyBqCbOqwbJVV1fLvDTiM1xgSOwE4EAANPgd3fQcFmThycQnCQ2NlDxpiAYolgxPmAwMrXiIsIZVy/ROavsw5jY0zgsEQQ1wsGTIaVr4IqU4d2Y83OEruhvTEmYFgiABh5IRRthR2LmTosFYDPN1jzkDEmMPg0EYhIgoi8LiLrRGStiExotF5E5BER2SQiK0VkrC/jadZRZ0JIJKycy9DusfSIi2CBDTdhjAkQvq4RPAx8qKrDgFHA2kbrpwGD3WkO8ISP42laeCwcNR1WvYl4apg6LJWFG/dQa6eRGmMCgM8SgYjEA5OAZwBUtUZVG5+XOQN4Xh3fAAki0tNXMbVo5EVQtRc2fMSUod0oq64jM6vIL6EYY0x78mWNoD+QDzwrIt+KyD9EJLrRNr2BHV7Ps91lBxCROSKSKSKZ+fk+arIZMAWiu8HKV5k4KIXQYLGrjI0xAcGXiSAEGAs8oapjgHLg5sM5kKo+paoZqpqRmpraljHuFxwCI2bCho+I8ZRw0uBU3vo2h5o6ax4yxnRtvkwE2UC2qi5yn7+Okxi85QB9vJ6nucv8Y9SFUF8Lq9/ksuP7kV9azUerd/ktHGOMaQ8+SwSqugvYISJD3UWnAGsabfY2cLl79tDxQLGq7vRVTAfVYySkHgUrXmXykFT6JkXx/NdZfgvHGGPag6/PGroWeElEVgKjgT+LyNUicrW7/n1gC7AJeBr4uY/jaZmIUyvIXkxQ0RZ+fHxflmQVsXZniV/DMsYYX/JpIlDV5W7b/khVPVtVi1T1SVV90l2vqvoLVR2oqiNUNdOX8bTKiAtoGHLigow+hIcE8fzX2/wdlTHG+IxdWdxYfG/ofxKsfJWEyFDOGtWLed/mUFxZ6+/IjDHGJywRNGXkRfuGnLh8QjqVtR7eWJrt76iMMcYnLBE05eiznCEnlr/IiLR4RvdJ4MVvtlFfr/6OzBhj2pwlgqaEx8Koi2DFXCjJ5fIJ/diyp5wvN+/xd2TGGNPmLBE058TfgtbDlw9z+oieJEWHWaexMaZLskTQnMR+Tq1g6XNEVOVz4fg+fLZ2Nzl7K/0dmTHGtClLBC056XfgqYWvHuXS4/oC8NI3ViswxnQtlghakjTAGX8o85+khZZzylHdeXXJDqrrPP6OzBhj2owlgoOZdD3UVsLXj3H5hH4UlNcw71v/DYdkjDFtzRLBwaQMhmPOg8VPc2KvIEb3SeD+jzdQXl3n78iMMaZNWCJojUnXQ205sugJbjvzaPJLq3l8wSZ/R2WMMW3CEkFrdDsKjp4Bi/7O2FQ4e3Qvnl64lR2FFf6OzBhjjpglgtaadANUl8Civ3PTtGEEi3DvB+v8HZUxxhwxSwSt1WMEDD0DvnmcnuG1XD15IO99t5NFWwr8HZkxxhwRSwSHYvINUFUM//1f5kwaQK/4CO58dw0eG4PIGNOJWSI4FL3GOCOTfvUokcWbuWnaMFbnltjIpMaYTs0SwaE67U8QFgXv/46zRvZkXL9E/vrRekqr7H4FxpjOqVWJQEReaM2ygBDTDU65DbZ+gax+k9umH82esmr+Nn+zvyMzxpjD0toawXDvJyISDIxr+3A6iXFXOs1EH93KqFThvLFp/PO/W9mUV+bvyIwx5pC1mAhE5BYRKQVGikiJO5UCecC/2yXCjigoGM54EMryYP6fuWnaUKLCg/ndv1ZQ56n3d3TGGHNIWkwEqnqPqsYC96lqnDvFqmqyqt7STjF2TL3HwvifwOKn6Fa2nrvOPoYVO/by5OfWRGSM6Vxa2zT0rohEA4jIj0XkQRHp58O4OoeT/wBRyfDudUw/pgdnjurFw59tZHVusb8jM8aYVmttIngCqBCRUcDvgM3A8z6LqrOITITT7oKcTPj2ee48azgJUWH87rUVNlS1MabTaG0iqFNVBWYAj6nq34BY34XViYy8EPpNhE9uJ7G+iHvPHcG6XaU88tlGf0dmjDGt0tpEUCoitwCXAe+JSBAQerCdRCRLRL4TkeUiktnE+ikiUuyuXy4itx1a+B2ACEz/X6irhrfmcMrQVC7ISOOJBZtZtr3I39EZY8xBtTYRXAhUA7NVdReQBtzXyn2nqupoVc1oZv1Cd/1oVb2zlcfsWFKHwrS/wJYF8OVD/HH60fSMj+T611ZQWWNNRMaYjq1VicD98n8JiBeR6UCVqlofgbexl8Pwc+E/dxGb/y33nT+SLXvK+cuHNkKpMaZja+2VxRcAi4GZwAXAIhE5vxW7KvCxiCwVkTnNbDNBRFaIyAciMrypDURkjohkikhmfn5+a0JufyJw5kMQnwav/4QTegcz64R0nvsqi49X7/J3dMYY0yxx+oAPspHICuAHqprnPk8FPlXVUQfZr7eq5ohIN+AT4FpV/cJrfRxQr6plInI68LCqDm7pmBkZGZqZ+b3uho4jeyn88zQYejrV5z7L+U9+w7aCct771Un0SYryd3TGmAAlIkuba6JvbR9BUEMScBW0Zl9VzXEf84C3gGMbrS9R1TJ3/n0gVERSWhlTx5Q2zhmLaO3bhK94nr9dMhYFfvHyMjul1BjTIbU2EXwoIh+JyCwRmQW8B7zf0g4iEi0isQ3zwGnAqkbb9BARceePdePp/Hd6mXAtDDwFPryFvnVbeWDmKFZmF3P3e2v9HZkxxnzPwcYaGiQiE1X1BuDvwEh3+hp46iDH7g78121WWgy8p6ofisjVInK1u835wCp3m0eAi7Q1bVUdXVAQnPN3iIiHf83itIFRXHVSf57/ehvvrMj1d3TGGHOAFvsIRORd4BZV/a7R8hHAn1X1TB/H9z0dvo/A29aF8PwMGDqN2vP/j4ueXsy6nSW8c+2JDEiN8Xd0xpgAciR9BN0bJwEAd1l6G8TWtfU/yRmCYt27hH75vzx2yRjCQoL4+UvL7PoCY0yHcbBEkNDCusi2DKTLOv4aGHEBzL+bnru/4KGLxrB+dyk3vL7C7nVsjOkQDpYIMkXkqsYLReSnwFLfhNTFiMCZD0OPEfDGVUxOLuHmHw3j3ZU7+f1b39EVukSMMZ1byEHW/wZ4S0QuZf8XfwYQBpzjy8C6lLAouPBFeGoKzL2En/30U8qq63j0P5uICgvhj9OPwj15yhhj2l2LiUBVdwMniMhU4Bh38Xuq+h+fR9bVJPaDmc/BC2fDvGu4bubzlFXX8c8vtxITHsx1pw31d4TGmAB1sBoBAKo6H5jv41i6vgGT4Qd/go9/j8y/i9vO+AOVNR4e+c8mIsNCuGbKQH9HaIwJQK1KBKYNTfgF7FkPCx9A6j3cffbtVNR4+MuH64gOD+byCen+jtAYE2AsEbQ3EZj+MASFwpcPEVxbyQMz/0xFjYfb/r2aiJBgLhjfx99RGmMCiCUCfwgKgjMegNBI+PoxQusqeeyiB5jz0nJuenMlIjAzw5KBMaZ9WCLwFxHnYrOQCFh4PxF11Tx16aNc9eJybnxjJUEinDcuzd9RGmMCgCUCfxKBU/4IoRHwn7uIqKvi6Uv/zk9eXMH1r69whiwaY8nAGONblgg6gkk3QGgUfHQrEdVl/OOiZ5n98lp+99oKgkSYMbq3vyM0xnRhrR2G2vjahF/AWY/ClvlEvnIOz1zQn2P7J/HbV5fzto1YaozxIUsEHcnYy+HCl2D3aqJeOINnz+lBRrqTDFblFPs7OmNMF2WJoKMZdjpc9haU5RH5/DSe+VE0CZGh3PbvVdTbIHXGGB+wRNAR9TsBZn8AqsS+Mp37jqtg2fa9vLEs29+RGWO6IEsEHVX34fCTjyE6lamL53Bhz13c+8E6iitq/R2ZMaaLsUTQkSX2gys/RGK6c1fVPURW5PLgJ+v9HZUxpouxRNDRxaTCJa8RWl/N6/GP8OY361idax3Hxpi2Y4mgM+g2DGY+S/fqLTwW/gR3zFtpHcfGmDZjiaCzGHQq8qO/MJlMTs19gje/zfF3RMaYLsISQWdy3Bx0/FX8LOQ91rz7GMWV1nFsjDlylgg6GfnRvZT2nsQt9U/x4isvUlXr8XdIxphOzhJBZxMcQuyPX6A4qg8/2XYDf7/3Ov61JAuP9RkYYw6TTxOBiGSJyHcislxEMptYLyLyiIhsEpGVIjLWl/F0GZEJpPziEyr6TObXnufo984FzH7wVT5buxtVSwjGmEPTHjWCqao6WlUzmlg3DRjsTnOAJ9ohnq4hphtJP3kdnfE4Y8JzebLsV8x/8R4u+vtXrNix19/RGWM6EX83Dc0AnlfHN0CCiPT0c0ydhwgy5lJCf7mI8AETuSv0Wa7bfRO/eHwet7y5ksLyGn9HaIzpBHydCBT4WESWisicJtb3BnZ4Pc92lx1AROaISKaIZObn5/so1E4svjdBl70J0x/i2NAtzI+4gW7LHuGH933MC19b/4ExpmW+TgQnqupYnCagX4jIpMM5iKo+paoZqpqRmprathF2FSKQcSXy80WEHnU6vw35F+8GX8+Cd57nzEf/y9Jthf6O0BjTQfk0EahqjvuYB7wFHNtokxzA+y7tae4yc7gS+sAF/weX/5tuiXE8E/YAvy++neuefGrTC5EAABgMSURBVItb3vyO0iq79sAYcyCfJQIRiRaR2IZ54DRgVaPN3gYud88eOh4oVtWdvoopoAyYglzzJZx2NyeErOeziJtIX3Yv5z74AfPX5/k7OmNMB+LLGkF34L8isgJYDLynqh+KyNUicrW7zfvAFmAT8DTwcx/GE3iCQ+GEXyLXLiVk5EzmhLzHG7U/5/Pn/8SNr2bakNbGGACks513npGRoZmZ37skwbTGzpXUf/QHgrI+J0t78HjIZUydMZsfHtOToCDxd3TGGB8SkaXNnMZviSDgqMKmT6l67xYi9m4ks34Ir0RdwuDjz+T8jD6kxIT7O0JjjA9YIjDf56mjbtnz1H52L5FVu1leP4An6s8h9KgzuOT4fkwYkIyI1RKM6SosEZjm1VXD8pep/fxBQku3s55+PFpzFhuTT2bOlCGcNboXocH+vu7QGHOkLBGYg/PUwarXqf/ifoIKNrI9KI2/Vp3D8tgpzJkyiAsy+hARGuzvKI0xh8kSgWm9eg+sfRtd8Bckfy3bQtK5u+IclkWewOyTBnDZ8f2IjQj1d5TGmENkicAcunoPrHoTXXAPUriZrWGD+Z+ys1kRPp5rpg7i8gnpVkMwphOxRGAOn6cOVr4Kn98Le7eTFTqYh8p/wOKoSVxz6tFcmNGHsBDrQzCmo7NEYI5cXQ2seBm+/hvs2UBhUBLPVJ/K53FncOWpGdapbEwHZ4nAtJ36etj8H/SbvyGb/0M1YbxRdyLvRZzBuGNP4sJj+9I7IdLfURpjGrFEYHwjby36zRPUr5hLsKea5fUDmes5mdJBZ3H+hGFMGpJKsF2xbEyHYInA+FZFIax8ldrF/yS0cANlRDKv7gQ+jTqDkRkncu7YNNJTov0dpTEBzRKBaR+qsGMRniXPwuq3CK6vZkX9AF72nEJu72mckTGY00f2JM5OPzWm3VkiMO2vsghWvErtkn8SWrCeCiJ4q24ir3Mqo4+bzK2nH2Wdy8a0I0sExn9UIXsJuvRZ9Ls3CfJUsbh+KO90/znXX3kp8VFWOzCmPVgiMB1D5V5YMZeq+X8lorqAT0KmcPRlD9C73yB/R2ZMl9dSIrC6uWk/kQlw/NVEXLeCnBE/Z1LdlyQ9O4HcebdBTbm/ozMmYFkiMO0vPJbe591D3uX/5evg8fRa/jCVD46BhQ/Cno3+js6YgGNNQ8avisprePCZ/+OsPU8xPmgDAJoyBBl2BgybDr3GQpD9XjHmSFkfgenQqus8PPDxBhYs/pbjahdxZtgyxrGaYPVATA8Y8kMY8iMYMBnC7HoEYw6HJQLTKVTXefhsbR6vL81m2fotTJblXBi7kvGebwmtK4fgcOg/yU0MP4SEvv4O2ZhOwxKB6XTySqqYtzyH1zKz2Za3l8nhG5ndbQMZ1YsIK8lyNuox0mk+Omo6dDsa7NaaxjTLEoHptFSVJVlFvLxoG++v2kVNXT1n9CrjJ6nrOar4cyJ2LUVQSOzvJIQh0yBtPISE+Tt0YzoUSwSmSygqr+GNZdm8vHg7W/Kd003Tw8v4ceIqTtbFpJcsJUhr0dBoJP1EGHgyDJwKKUOstmACnl8TgYgEA5lAjqpOb7RuFnAfkOMuekxV/9HS8SwRGFVlc34Z327fy4rsvSzfsZd1O0uJrC/nhKDVTA1dzaSQVfTy5AJQHdUDGTiVsCGnwoApEJ3i1/iN8Qd/J4LrgAwgrplEkKGqv2zt8SwRmKZU1XpYlVPM2p0lbNhdxobdpZTt3sKI6mWcGPQdE4NWkyhlAFQmDydiyCnIoJOh7/EQavdPMF1fS4kgxMcvnAacAdwNXOfL1zKBLSI0mIz0JDLSk7yWTmBP2fms21nKPzbtJmfdInrt+ZqT8laRUfA4oV8/gicoHPoeT/CgqU5toccou27BBByfJgLgIeBGILaFbc4TkUnABuC3qrrDxzGZAJISE86Jg8M5cXAKTBtOfumlLNyYz63rtlG1cSGjapZz0tbVDM36HACNTEL6n+Scptp/MiQPsv4F0+X5rGlIRKYDp6vqz0VkCnB9E01DyUCZqlaLyM+AC1X15CaONQeYA9C3b99x27Zt80nMJrDUeepZtLWQD1btZMl36xhWuYxJIauZErqGZE8+ADWR3aD/JMIGTXaSQ2K6f4M25jD5pY9ARO4BLgPqgAggDnhTVX/czPbBQKGqxrd0XOsjML7gqVeWbivi/e928vWmPXgKtpDBKk4IWs2EoDWkSjEAJeE9qU6bSMLwkwkdOBni0/wcuTGt4/fTR1uoEfRU1Z3u/DnATap6fEvHskRg2oOnXskuqmBzfhmbd5dRsmMVETlf0b9sGccHrSHJ7XjeG5FGWNpoonod7ZymmjLYmWwoDNPB+K2zuJlg7gQyVfVt4FcichZOraEQmNXe8RjTlOAgoV9yNP2Sozl5WHdgIDCDovIalmzdQ9aaTMhaSL/Sbxm6cSl9N71PEPX7DxDfF/qMh74TnDOTuh0NQcH+Ko4xLbILyow5Ajl7K7lt3ioWrsvh1O5l3Dw+mL712bB7NWxfBKXOtQyEx0GfY52k0HcC9B5np62aduX3pqG2ZInAdDSqygerdnH726spKKtm9sT+/PYHQ4gOC4biHbDta9j+NWz/BvLXOjsFhULPUfsTQ9/j7UI341OWCIxpB8WVtfzlw3W8vGg73ePCGds3kb5JUfRJiqKvO/UKryIsd4mTGHYsgpyl4KlxDpA6DPqdAP0mOo9xvfxbINOlWCIwph0tySrkyQWb2VpQTnZhJTWe/X0HwUFCenIUg7vFMqR7DENSwjiGzaSVriBkh1trqCl1Nk5Mh56jofsx0P1op58hoZ9d8GYOiyUCY/ykvl7JK61me2EF2wsr2LqnjI27y9iUV0ZWQTn17r9faLAwpHssI3tGc2LcbkZ5VtFz73KC81dBUdb+A4bFODWH5EGQNMCZkt3HyES/lNF0DpYIjOmAqmo9bN1TzobdpazbVcqqnGJW5RRTVFELQJBAWmIUw5KCGB+9m+HBO+hXl0Vy5RYiirOgJPvAA0alQI8R0HOkc6+GHiMheaCdrWQASwTGdBqqys7iKicp5JawJd+pOWzNL6e8xrNvu75JUZw8KJYf9KxibEwhkaXbIH8d7FwJeWuh3kkmhEY7yaHXGOg91nlMGmjNSwHIEoExnZyqkl9azZY95azdWcJ/N+7h6y0FVNR4CAkSxvZLZHSfBMJDggjDQ7eaLLqXb6Bb2Tp6Vm4goXgtUlvhHCw8zjljqecop9bQcyQkD4bgdr+syLQjSwTGdEHVdR6Wbiviiw17+GJDPhvzSqn1NP3/HIyHU1OL+WFCLmNCtpJWsZbQgnVQV+VsEBLpdEj3HAW9xjrXOaQOtWalLsQSgTEBQlXx1Ct19Uqtp55aj7JxdylLsgpZtLWQpduKqHCbmBIjhBPiCzk2IpujJIt+NZtILl1HSK171lJYzP4mpZ6jIGWo0+dgF8J1SpYIjDGAM+Lqmp0lLMkqImtPOdsKK9hRWEF2UQW1HkWo55KBtdxwTDkJRSud6xx2fbf/WgcEEvo6tYWUIc7ZSymDnceY7jZkdwfWocYaMsb4T0hwECPTEhiZlnDAck+9srO4kg++28UDn6zn7Zxo/njGr5g5LQ3x1ED+eijYCHs2OvN7NsLWL/Y3LYHT95A80E0OQ51kkToMkvpDcGg7l9QcCqsRGGMOkLWnnJveWMmirYWcNDiFe84dQVpi1Pc3rK93TmHdsxEKNjuJomCT87zY6/5SQaFOckgd4pyxlDzQvQZiIMR0s1pEO7GmIWPMIamvV15atI17PliHAFdO7E/vxEgSo0KJjwwjMTqUxKgwUmLCCQ5q4ou8usxJDPnrndNa8zfAnvXOxXH1dfu3C4uBuN4QHus1xTmPSf2dTuvux0BoRHsVvcuyRGCMOSw7Civ4/bxVfLEhv8n1YcFB9E+JZlD3GAalxjComzP1TYoiOryJlmdPHRRvh8ItULAFCjdDSS7UlEF1qZNAqkuhqnj/UBtBodDjGPdsprHO0BuxPZ0+ifAY3xW+i7FEYIw5IpU1HvZW1lBUXsveihqKKmoprKghu7CCTXllbMovY3thBd5fJ0nRYaQlRrpTFN3jIggNFoJECA4SgkUQgeSYMMb0SSQxOuzAFy3JdTqrc5ZCzjLI/RaqSw7cJiwWYns4U3SKc3V1dCpEJzvzkQkQEgHBYc5jSLjzGJ3izAcQSwTGGJ9rGDJjU14ZO4oqyC6qdCdnvqauvsX9B6ZGk9EviXHpiWT0S6RnvHOaakMXglBPcNFWQspyoXQ3lO6E0l37Hyv2QPkeqNp78GAlyDn7Kcnt3E4e6Mwn9HGbqrpeTcMSgTHGr+rrlZKqWurqlfp6xeNe76AKuXsrydxWxFJ3Kq6sbfFYfZIiGd4znmN6xzG8VzzDe8XRLc6rD8FTCxUFUJ7vNDHVVTunv9ZVOfO1lU5to3Cz07ldsGV/M1SDiHiIS3OGAk/oA4n99w/yl5gOYU10nndwdvqoMcavgoKEhKiwJtf1SYriuAHJgJMwNueXsXRbEXsra1EFRfc1OdXU1bMpr4zVucV8uHrXvmPER4YSFRZMWEgQYcFBzmNIEMN7xXPTj4YRG9HC6auqUJbn9FsUZztnQhXnQEmO8zwnEyqLDtwn1k0Q0amNmqRSnOQR19t57CSnzVqNwBjTKZVU1bI2t4TVuSVs3VNOdZ2H6rp6atypstbDN1sK6JMUxaMXj/netROHpLIICrc6yaJwq9vJneM0RZXvcZqltHHTlzh9F/FpTmLYlzSS90/RqU6nd1SSz0+jtaYhY0xAWpJVyK9f+Zb8smpu+tEwZk/sT1BTp7seqfp6J1mU5zl9FsXZ7pTjXFNRkus2VTXTfxEU6iSEmG5O8ojr5fRhJPRzHhPTnftNHEGysERgjAlYeytquPH1lXy8ZjdThqbywMxRJMf46YwhTx1UFrp9GHuc5FCWB2W7nA7whseSbKd/w1tYLEz8NUy+4bBe2voIjDEBKyEqjL9fNo4XvtnGXe+tZdrDC5kxuhdBQe6prCIEiTP8RlJ0GN1iw+keF0G3uHBSYsIJDW7DezcEhzi/+mO6HXzbyr2wd7s7bYOibdBtWNvF4sVqBMaYgLEmt4QbXl/Blvxy6lXdyRlrqSkikBztXEGdGus1xYSTFB1GfGToAVNcZCgRoR1z6G6rERhjDHB0rzje+9VJTa6r89RTUF5DXkk1u0uqyCvd/7inrNq5MVB+Ofml1dR4mr8mokdcBMN7xXF0rziG93JOcU1LjEREUFVqPUplrYfqWg8IpMaEI34eb8nniUBEgoFMIEdVpzdaFw48D4wDCoALVTXL1zEZY0xjIcFBdI+LoHtcBCOIb3Y7VaWkqo7C8hqKK2v3TSWVzlXXm/PLWZ1bzPz1eTRUNKLCghGgstZD48pHTHgIA7sdOERH74RI4qOcWkZ0WLDPE0V71Ah+DawF4ppY9xOgSFUHichFwF+AC9shJmOMOSwisq8pqCVVtR7W7SplTW4JG/NKCRIhMjSYiNAgIkKDiQgNps5Tzxb3auz/bsrnjWXZ3ztOcJAQFxFCfGQoPz6+Hz89aUCbl8mniUBE0oAzgLuB65rYZAZwhzv/OvCYiIh2to4LY4xpJCI0mNF9Ehjdp/XXL5RU1bI5r4zdJVUH1DacGkcdqbG+OdvJ1zWCh4Abgdhm1vcGdgCoap2IFAPJwB7vjURkDjAHoG/fvj4L1hhj/CkuIpQxfRPb/XXb8LyoA4nIdCBPVZce6bFU9SlVzVDVjNTU1DaIzhhjTAOfJQJgInCWiGQBc4GTReTFRtvkAH0ARCQEiMfpNDbGGNNOfJYIVPUWVU1T1XTgIuA/qvrjRpu9DVzhzp/vbmP9A8YY047a/ToCEbkTyFTVt4FngBdEZBNQiJMwjDHGtKN2SQSqugBY4M7f5rW8CpjZHjEYY4xpmi/7CIwxxnQClgiMMSbAWSIwxpgA1+lGHxWRfGBbKzZNodGFaV1QIJQRAqOcVsauo6OWs5+qNnkhVqdLBK0lIpnNDbnaVQRCGSEwymll7Do6YzmtacgYYwKcJQJjjAlwXTkRPOXvANpBIJQRAqOcVsauo9OVs8v2ERhjjGmdrlwjMMYY0wqWCIwxJsB1uUQgIj8SkfUisklEbvZ3PIdKRLJE5DsRWS4ime6yJBH5REQ2uo+J7nIRkUfcsq4UkbFex7nC3X6jiFzR3Ou1FxH5p4jkicgqr2VtVi4RGee+b5vcfdv9buDNlPEOEclxP8/lInK617pb3HjXi8gPvZY3+TcsIv1FZJG7/FURCWu/0u2LoY+IzBeRNSKyWkR+7S7vap9lc+XsUp/nPqraZSYgGNgMDADCgBXA0f6O6xDLkAWkNFr2V+Bmd/5m4C/u/OnAB4AAxwOL3OVJwBb3MdGdT/RzuSYBY4FVvigXsNjdVtx9p3WQMt4BXN/Etke7f5/hQH/37za4pb9h4DXgInf+SeAaP5SxJzDWnY8FNrhl6WqfZXPl7FKfZ8PU1WoExwKbVHWLqtbg3BBnhp9jagszgP9z5/8PONtr+fPq+AZIEJGewA+BT1S1UFWLgE+AH7V30N5U9Qucoca9tUm53HVxqvqNOv9Vz3sdq900U8bmzADmqmq1qm4FNuH8/Tb5N+z+Kj4Z597ecOD71W5UdaeqLnPnS4G1OLec7WqfZXPlbE6n/DwbdLVEsO8eyK5sWv7wOiIFPhaRpeLcqxmgu6rudOd3Ad3d+ebK21neh7YqV293vvHyjuKXbrPIPxuaTDj0MiYDe1W1rtFyvxGRdGAMsIgu/Fk2Kid0wc+zqyWCruBEVR0LTAN+ISKTvFe6v5K63Dm/XbVcwBPAQGA0sBN4wL/htA0RiQHeAH6jqiXe67rSZ9lEObvk59nVEsG+eyC70txlnYaq5riPecBbOFXL3W6VGfcxz928ufJ2lvehrcqV4843Xu53qrpbVT2qWg88jfN5wqGXsQCnWSWk0fJ2JyKhOF+OL6nqm+7iLvdZNlXOrvh5QtdLBEuAwW5vfBjOrS/f9nNMrSYi0SIS2zAPnAas4sB7O18B/Nudfxu43D0z43ig2K2efwScJiKJbtX1NHdZR9Mm5XLXlYjI8W7b6+Vex/Krhi9H1zk4nyc4ZbxIRMJFpD8wGKeTtMm/YfdX9nyce3vDge9Xu3Hf32eAtar6oNeqLvVZNlfOrvZ57uOvXmpfTThnKWzA6an/vb/jOcTYB+CcVbACWN0QP0574mfARuBTIMldLsDf3LJ+B2R4HWs2TofVJuDKDlC2V3Cq0rU47aE/actyARk4/5Sbgcdwr5rvAGV8wS3DSpwvi55e2//ejXc9XmfGNPc37P59LHbL/i8g3A9lPBGn2WclsNydTu+Cn2Vz5exSn2fDZENMGGNMgOtqTUPGGGMOkSUCY4wJcJYIjDEmwFkiMMaYAGeJwBhjApwlAhNwRKTMfUwXkUva+Ni3Nnr+VVse3xhfsERgAlk6cEiJwOtK0OYckAhU9YRDjMmYdmeJwASye4GT3HHlfysiwSJyn4gscQcV+xmAiEwRkYUi8jawxl02zx0YcHXD4IAici8Q6R7vJXdZQ+1D3GOvEmes/Qu9jr1ARF4XkXUi8pJ7VSsicq844+GvFJH72/3dMQHjYL9ujOnKbsYZW346gPuFXqyq40UkHPhSRD52tx0LHKPOEMMAs1W1UEQigSUi8oaq3iwiv1TV0U281rk4A5WNAlLcfb5w140BhgO5wJfARBFZizOEwTBVVRFJaPPSG+OyGoEx+52GMy7Ocpwhh5NxxowBWOyVBAB+JSIrgG9wBhUbTMtOBF5RZ8Cy3cDnwHivY2erM5DZcpwmq2KgCnhGRM4FKo64dMY0wxKBMfsJcK2qjnan/qraUCMo37eRyBTgVGCCqo4CvgUijuB1q73mPUCIOuPUH4tz45LpwIdHcHxjWmSJwASyUpzbEDb4CLjGHX4YERnijgLbWDxQpKoVIjIM57aKDWob9m9kIXCh2w+RinNby8XNBeaOgx+vqu8Dv8VpUjLGJ6yPwASylYDHbeJ5DngYp1lmmdthm0/Ttw/8ELjabcdfj9M81OApYKWILFPVS72WvwVMwBlZVoEbVXWXm0iaEgv8W0QicGoq1x1eEY05OBt91BhjApw1DRljTICzRGCMMQHOEoExxgQ4SwTGGBPgLBEYY0yAs0RgjDEBzhKBMcYEuP8HkYMPGwNdbkgAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ePBg2CpQWF8s"
      },
      "source": [
        "Let’s quickly save our trained model:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "82TgU7-LWF8s"
      },
      "source": [
        "PATH = '/content/drive/My Drive/Transformer1.pth'\n",
        "torch.save(model.state_dict(), PATH)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bd74kNjnWF8s",
        "outputId": "1b9e413d-40b4-47cb-fd25-2036c2921f8e"
      },
      "source": [
        "print(train_losses)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[7.043605698585511, 6.296885191917419, 5.99202653503418, 5.781997014045715, 5.628804269790649, 5.499716843605041, 5.384029871940613, 5.291666975975037, 5.203094429016113, 5.0565361375808715, 5.009000703811646, 4.9567625484466555, 4.899050359725952, 4.863669639587402, 4.82200162601471, 4.776107933998108, 4.740412371635437, 4.707698857307434, 4.632751619338989, 4.59986457824707, 4.564599791526795, 4.545856578826904, 4.531093785285949, 4.506484023094178, 4.477768167495728, 4.463965636253357, 4.442565082550049, 4.387112852096558, 4.375498162269592, 4.3609973926544185, 4.3438606424331665, 4.330902749061584, 4.306575284004212, 4.304798285484314, 4.283659282684326, 4.271893107891083, 4.2141468358039855, 4.207668862819672, 4.1936048374176025, 4.182595546722412, 4.1816437344551085, 4.161390475273132, 4.161115400791168, 4.144227269649505, 4.141771843910218, 4.077436944961548, 4.076544713973999, 4.0678434686660765, 4.059634812355042, 4.05384567117691, 4.037088141918183, 4.0403606591224674, 4.026344301223755, 4.020610077381134]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5ZwxKBBIWF8t",
        "outputId": "ba88348d-18d6-4fc8-b189-34376b6e54d8"
      },
      "source": [
        "print(validation_losses)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[6.477477967739105, 6.114445004890214, 5.872668937960667, 5.693810649772189, 5.549676311549856, 5.430023546539136, 5.320086118000657, 5.235531264276647, 5.155722340541099, 5.051319510189455, 4.991950946067696, 4.939749171484762, 4.891047031132143, 4.848623677865783, 4.80355553662599, 4.764846461032754, 4.729845322779755, 4.694820641581692, 4.644868017132603, 4.617324741918649, 4.59135195390502, 4.563618867254969, 4.539941929169555, 4.516147624200849, 4.492726012841979, 4.470247595167872, 4.447953905632247, 4.434103411525043, 4.413478537281947, 4.392982181328446, 4.3716673228278085, 4.35614977398915, 4.343683045301864, 4.326986000609042, 4.3090091405519795, 4.292229873030933, 4.273640993370939, 4.259844438798392, 4.24659691283952, 4.232274355283424, 4.219687797685168, 4.214949277799521, 4.196480119406288, 4.187565350710456, 4.180212653839766, 4.177562744759801, 4.1647023945602015, 4.154781518142615, 4.142465530046776, 4.135001771485627, 4.124816796672878, 4.117777992540331, 4.109282061680039, 4.098103115362908]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pKb1iEhLgB8j"
      },
      "source": [
        "## Test "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 267,
          "referenced_widgets": [
            "09e08c49f9544dd094c8c66856bc16ab",
            "1d938503ada349d49db03e07076aa4e5",
            "0eb930bff3464e078ba75074c5785f74",
            "71b30636499f406c8e462e6eb188b999",
            "2c798f31332c4b759b2fa7febfce93bd",
            "79c02617407b42d1864f66eddc3a25f2",
            "e35c15bdd93e40ef85fa396608bceaed",
            "3b17e7fbcb104130a86a2a7d074c124c"
          ]
        },
        "id": "Ecu2IUFfXLat",
        "outputId": "20405f8f-d7fc-4e68-d49c-f551a8e7a81d"
      },
      "source": [
        "torch.manual_seed(2045)\n",
        "\n",
        "pad_idx = 0\n",
        "\n",
        "# Parameters\n",
        "params = {'batch_size': 1,\n",
        "          'shuffle': False,\n",
        "          'num_workers':2,\n",
        "          'collate_fn' : BatchPadCollate(pad_idx=pad_idx)\n",
        "          }\n",
        "\n",
        "# Training and validation Data Generators\n",
        "test_set0 = AFEC_Dataset(en_test_dic,fa_test_dic0,partition['test'],en_vocab_dic, fa_vocab_dic)\n",
        "test_generator0 = torch.utils.data.DataLoader(test_set0, **params)\n",
        "test_set1 = AFEC_Dataset(en_test_dic,fa_test_dic0,partition['test'],en_vocab_dic, fa_vocab_dic)\n",
        "test_generator1 = torch.utils.data.DataLoader(test_set1, **params)\n",
        "test_set2 = AFEC_Dataset(en_test_dic,fa_test_dic0,partition['test'],en_vocab_dic, fa_vocab_dic)\n",
        "test_generator2 = torch.utils.data.DataLoader(test_set2, **params)\n",
        "test_set3 = AFEC_Dataset(en_test_dic,fa_test_dic0,partition['test'],en_vocab_dic, fa_vocab_dic)\n",
        "test_generator3 = torch.utils.data.DataLoader(test_set3, **params)\n",
        "\n",
        "\n",
        "# Hyperparameters\n",
        "embed_dim = 256\n",
        "num_heads = 8\n",
        "dim_key = 64\n",
        "dim_value = 64\n",
        "dim_inner = 1024\n",
        "num_layers = 3\n",
        "src_vocab_size = len(en_vocab_dic.get_str_2_idx())\n",
        "trg_vocab_size = len(fa_vocab_dic.get_str_2_idx())\n",
        "learning_rate = 1e-4\n",
        "\n",
        "kwargs = {'p_dropout': 0.1,\n",
        "          'kdim': dim_key,\n",
        "          'vdim':dim_value,\n",
        "          }\n",
        "# Model\n",
        "# PATH = '/content/drive/My Drive/Transformer1.pth'\n",
        "# model = Transformer(num_layers, embed_dim, dim_inner, num_heads, src_vocab_size,trg_vocab_size, device,**kwargs).to(device)\n",
        "# model.load_state_dict(torch.load(PATH))\n",
        "\n",
        "test_dataiter0 = iter(test_generator0)\n",
        "test_dataiter1 = iter(test_generator1)\n",
        "test_dataiter2 = iter(test_generator2)\n",
        "test_dataiter3 = iter(test_generator3)\n",
        "\n",
        "list_of_references = []\n",
        "hypothesises = []\n",
        "bleu_scores = []\n",
        "nist_scores = []\n",
        "\n",
        "# Loop over batches\n",
        "for num_batch in tqdm(range(len(test_generator0))):\n",
        "    # Test \n",
        "    with torch.set_grad_enabled(False):\n",
        "        # Transfer to GPU\n",
        "        batch_src0, batch_trg0 = test_dataiter0.next() \n",
        "        batch_src0, batch_trg0 = batch_src0.to(device), batch_trg0.to(device)\n",
        "        batch_src1, batch_trg1 = test_dataiter1.next() \n",
        "        batch_src1, batch_trg1 = batch_src1.to(device), batch_trg1.to(device)\n",
        "        batch_src2, batch_trg2 = test_dataiter2.next() \n",
        "        batch_src2, batch_trg2 = batch_src2.to(device), batch_trg2.to(device)\n",
        "        batch_src3, batch_trg3 = test_dataiter3.next() \n",
        "        batch_src3, batch_trg3 = batch_src3.to(device), batch_trg3.to(device)\n",
        "\n",
        "        # Model computations\n",
        "        # forward + backward + optimize\n",
        "        batch_outputs1 = model(batch_src0, batch_trg0[: , :-1])\n",
        "        batch_outputs = model.translate_sentence(batch_src0, batch_trg0.shape[1] + 5, 1, 2)\n",
        "        batch_outputs = torch.Tensor(batch_outputs)\n",
        "\n",
        "        references = [fa_vocab_dic.IndexDecoding(batch_trg0[0,:]),fa_vocab_dic.IndexDecoding(batch_trg1[0,:]),\n",
        "                      fa_vocab_dic.IndexDecoding(batch_trg2[0,:]),fa_vocab_dic.IndexDecoding(batch_trg3[0,:])]\n",
        "        list_of_references.append(references)\n",
        "        \n",
        "        hypothesises.append(fa_vocab_dic.IndexDecoding(batch_outputs))\n",
        "        bleu_scores.append(sentence_bleu(references,fa_vocab_dic.IndexDecoding(batch_outputs)))\n",
        "        nist_scores.append(sentence_nist(references,fa_vocab_dic.IndexDecoding(batch_outputs)))\n",
        "        \n",
        "        # print(batch_src0.shape, batch_trg0.shape, len(batch_outputs))\n",
        "        # print(en_vocab_dic.IndexDecoding(batch_src0[0,:])[1:-1])\n",
        "        # print(fa_vocab_dic.IndexDecoding(batch_trg0[0,:])[1:-1])\n",
        "        # print(fa_vocab_dic.IndexDecoding(batch_outputs)[1:-1])\n",
        "        # if num_batch == 5:\n",
        "        #   break\n",
        "\n",
        "bleu_score = corpus_bleu(list_of_references, hypothesises) \n",
        "nist_score = corpus_nist(list_of_references, hypothesises, n = 5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "09e08c49f9544dd094c8c66856bc16ab",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=250.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/nltk/translate/bleu_score.py:490: UserWarning: \n",
            "Corpus/Sentence contains 0 counts of 3-gram overlaps.\n",
            "BLEU scores might be undesirable; use SmoothingFunction().\n",
            "  warnings.warn(_msg)\n",
            "/usr/local/lib/python3.7/dist-packages/nltk/translate/bleu_score.py:490: UserWarning: \n",
            "Corpus/Sentence contains 0 counts of 4-gram overlaps.\n",
            "BLEU scores might be undesirable; use SmoothingFunction().\n",
            "  warnings.warn(_msg)\n",
            "/usr/local/lib/python3.7/dist-packages/nltk/translate/bleu_score.py:490: UserWarning: \n",
            "Corpus/Sentence contains 0 counts of 2-gram overlaps.\n",
            "BLEU scores might be undesirable; use SmoothingFunction().\n",
            "  warnings.warn(_msg)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MVYP7aJ3Sqf_",
        "outputId": "29746be9-4ea5-42d4-a55a-231f50b6e87e"
      },
      "source": [
        "print(bleu_score,nist_score)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.06216927618083542 0.0010401418852129015\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kttGBqpwocYQ",
        "outputId": "7a2f0610-0ea5-4e43-c5c8-43b1b7c97016"
      },
      "source": [
        "print(np.mean(bleu_scores),np.mean(nist_scores))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.470767768389585 0.07479292648936205\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pusMTedwTbKu",
        "outputId": "d1d58d04-0609-41f4-a950-5119dad92e35"
      },
      "source": [
        "print(fa_vocab_dic.IndexDecoding(torch.max(batch_outputs1, dim = 2)[1][0,:]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['بله', 'ما', 'ساعت', 'عصر', 'امروز', 'که', 'ما', 'چیز', 'درباره', 'کنیم', 'ما']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 906,
          "referenced_widgets": [
            "a2808f06173f427eb9d8be18570b68a8",
            "0894d23cd20946e797cb1757aa1a529c",
            "7064339856d040f68e789713d52b8247",
            "a7f3dceb2ece4ca8982b4a4d1a50a426",
            "96b8571b6e3e4b7190ebc2665a3d1101",
            "b106ad01751d413a9bc7f64410cfa383",
            "c7157f5c93f24b399c2546952ffe985b",
            "5fb185d7b19941aaae13526991e3aa30"
          ]
        },
        "id": "w3NS7SNTgUJO",
        "outputId": "2ff8cf62-5eb8-4b76-a7d9-ee774f53b6cd"
      },
      "source": [
        "torch.manual_seed(2045)\n",
        "\n",
        "pad_idx = 0\n",
        "\n",
        "# Parameters\n",
        "params = {'batch_size': 1,\n",
        "          'shuffle': True,\n",
        "          'num_workers':2,\n",
        "          'collate_fn' : BatchPadCollate(pad_idx=pad_idx)\n",
        "          }\n",
        "\n",
        "# Training and validation Data Generators\n",
        "test_set = AFEC_Dataset(en_test_dic,fa_test_dic0,partition['test'],en_vocab_dic, fa_vocab_dic)\n",
        "test_generator = torch.utils.data.DataLoader(test_set, **params)\n",
        "\n",
        "# Hyperparameters\n",
        "embed_dim = 256\n",
        "num_heads = 8\n",
        "dim_key = 64\n",
        "dim_value = 64\n",
        "dim_inner = 1024\n",
        "num_layers = 3\n",
        "src_vocab_size = len(en_vocab_dic.get_str_2_idx())\n",
        "trg_vocab_size = len(fa_vocab_dic.get_str_2_idx())\n",
        "learning_rate = 1e-4\n",
        "num_iterations = 10\n",
        "\n",
        "kwargs = {'p_dropout': 0.1,\n",
        "          'kdim': dim_key,\n",
        "          'vdim':dim_value,\n",
        "          }\n",
        "# Model\n",
        "# PATH = '/content/drive/My Drive/Transformer1.pth'\n",
        "# model = Transformer(num_layers, embed_dim, dim_inner, num_heads, src_vocab_size,trg_vocab_size, device,**kwargs).to(device)\n",
        "# model.load_state_dict(torch.load(PATH))\n",
        "\n",
        "test_dataiter = iter(test_generator)\n",
        "\n",
        "# Loop over epochs\n",
        "for itr in tqdm(range(num_iterations)):\n",
        "    \n",
        "    # Test\n",
        "    batch_src, batch_trg = test_dataiter.next() \n",
        "    \n",
        "    with torch.set_grad_enabled(False):\n",
        "\n",
        "\n",
        "        # Model computations\n",
        "        # forward + backward + optimize\n",
        "        \n",
        "        batch_outputs = model.translate_sentence(batch_src, batch_trg.shape[1] + 5, 1, 2)\n",
        "        batch_outputs = torch.Tensor(batch_outputs)\n",
        "\n",
        "        print(\"Sentence {}:\".format(itr + 1))\n",
        "        print(\" \".join(en_vocab_dic.IndexDecoding(batch_src[0,:])).replace(\"<sos>\", \"\").replace(\"<eos>\", \".\"))\n",
        "        print(\" \".join(fa_vocab_dic.IndexDecoding(batch_trg[0,:])).replace(\"<sos>\", \"\").replace(\"<eos>\", \"\").replace(\"<unk>\",\"ن.ش\"))\n",
        "        print(\" \".join(fa_vocab_dic.IndexDecoding(batch_outputs)).replace(\"<sos>\", \"\").replace(\"<eos>\", \"\").replace(\"<unk>\",\"ن.ش\"))\n",
        "        print(\"-\"*50)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a2808f06173f427eb9d8be18570b68a8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Sentence 1:\n",
            " once again shall we stay at the hotel <unk> .\n",
            " یکبار دیگر آیا قرار است در هتل ن.ش بمانیم \n",
            " وقتی که وقتی در هتل هتل هتل ن.ش \n",
            "--------------------------------------------------\n",
            "Sentence 2:\n",
            " fine let us go for a little walk in hamburg .\n",
            " خوب است اجازه دهید برای یک کوتاه در هامبورگ برویم \n",
            " اجازه دهید برای اینکه اجازه بدهید به ن.ش ن.ش برای ن.ش به ن.ش \n",
            "--------------------------------------------------\n",
            "Sentence 3:\n",
            " yes i have already booked the hotel .\n",
            " بله من در حال حاضر هتل را رزرو \n",
            " بله هتل ن.ش را رزرو \n",
            "--------------------------------------------------\n",
            "Sentence 4:\n",
            " in the evening we should watch the newest movie with arnold schwarzenegger .\n",
            " عصر باید جدیدترین فیلم آرنولد ن.ش را تماشا کنیم \n",
            " ما باید با ن.ش در عصر عصر ن.ش در عصر با ن.ش \n",
            "--------------------------------------------------\n",
            "Sentence 5:\n",
            " where should we meet .\n",
            " کجا باید ملاقات کنیم \n",
            " کجا باید ببینیم کجا باید ببینیم و باید همدیگر را ببینیم\n",
            "--------------------------------------------------\n",
            "Sentence 6:\n",
            " hello i would prefer the intercontinental .\n",
            " سلام من ن.ش را ترجیح میدادم \n",
            " سلام من ترجیح میدهم که ترجیح میدهم \n",
            "--------------------------------------------------\n",
            "Sentence 7:\n",
            " i have asked when we will arrive in hanover .\n",
            " من چه وقت به هانوور خواهیم رسید \n",
            " وقتی وقتی از من پرسید وقتی که به ن.ش کمک ن.ش \n",
            "--------------------------------------------------\n",
            "Sentence 8:\n",
            " fine let us meet at the station at nine oclock on thursday morning .\n",
            " خوب است اجازه دهید ساعت نه صبح ن.ش در ایستگاه ملاقات کنیم \n",
            " اجازه دهید ساعت نه صبح ساعت نه صبح در ایستگاه های ن.ش ملاقات کنیم \n",
            "--------------------------------------------------\n",
            "Sentence 9:\n",
            " yes i have got a timetable when would you like to go .\n",
            " بله من یک جدول زمانی دارم چه وقت دوست داری بروی \n",
            " بله من وقتی که شما را به شما گفتم بله \n",
            "--------------------------------------------------\n",
            "Sentence 10:\n",
            " i agree everything okay .\n",
            " موافقم همه چیز مرتب است \n",
            " خوب است خوب من همه چیز را به توافق موافقم \n",
            "--------------------------------------------------\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iVOCX2cU0yG4"
      },
      "source": [
        "# Part 2:"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VgD4EzgOCpGS"
      },
      "source": [
        "## Preprocessing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9eO1o8dCCpGT"
      },
      "source": [
        "### Load the dataset to colab directory "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9HQvI3xuCpGT"
      },
      "source": [
        "For the sake of speed, copy the dataset files to the root directory of notebook and unziped them. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UbGlO7RoCpGT"
      },
      "source": [
        "!cp /content/drive/My\\ Drive/AFEC-merged-all.zip /content/AFEC-merged-all.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xFt-PpxzCpGU"
      },
      "source": [
        "!cp /content/drive/My\\ Drive/Test.zip /content/Test.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vIWUsSS7CpGU",
        "outputId": "dc79d45a-9a64-41ff-e40c-ff547e6fdaf7"
      },
      "source": [
        "!unzip /content/AFEC-merged-all.zip"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/AFEC-merged-all.zip\n",
            "  inflating: AFEC-merged.en          \n",
            "  inflating: AFEC-merged.fa          \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3_cXmLzRCpGV"
      },
      "source": [
        "os.rename('/content/AFEC-merged.en',\"/content/AFEC-merged_en.txt\")\n",
        "os.rename('/content/AFEC-merged.fa',\"/content/AFEC-merged_fa.txt\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IQdc709pCpGW",
        "outputId": "ccfb0e65-47a3-404e-a4a7-7ff36f19f536"
      },
      "source": [
        "!unzip /content/Test.zip"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/Test.zip\n",
            "   creating: Test/\n",
            "  inflating: Test/test.en            \n",
            "  inflating: Test/test.fa0           \n",
            "  inflating: Test/test.fa1           \n",
            "  inflating: Test/test.fa2           \n",
            "  inflating: Test/test.fa3           \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5u2fCnD0CpGW"
      },
      "source": [
        "os.rename('/content/Test/test.en',\"/content/test_en.txt\")\n",
        "os.rename('/content/Test/test.fa0',\"/content/test_fa0.txt\")\n",
        "os.rename('/content/Test/test.fa1',\"/content/test_fa1.txt\")\n",
        "os.rename('/content/Test/test.fa2',\"/content/test_fa2.txt\")\n",
        "os.rename('/content/Test/test.fa3',\"/content/test_fa3.txt\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9fMbZjIu-Efa"
      },
      "source": [
        "### Vocabulary2 "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eFclS4hY-Efa"
      },
      "source": [
        "class Vocabulary_and_Dictionary2():\n",
        "\n",
        "    def __init__(self, freq_threshold = 2):\n",
        "        self.freq_threshold = freq_threshold\n",
        "        \n",
        "        self.frequencies = {}\n",
        "        self.idx_2_str = {0: \"<pad>\", 1: \"<sos>\", 2: \"<eos>\", 3: \"<unk>\"}\n",
        "        self.str_2_idx = {\"<pad>\": 0, \"<sos>\": 1, \"<eos>\": 2, \"<unk>\": 3}\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.idx_2_str)\n",
        "\n",
        "    def preprocess(self,text):\n",
        "        # prepare translation table for removing punctuation\n",
        "        table = str.maketrans('', '', string.punctuation)\n",
        "\n",
        "        # tokenize on white space\n",
        "        text = text.split()\n",
        "        # convert to lower case\n",
        "        text = [word.lower() for word in text]\n",
        "        # remove punctuation from each token\n",
        "        text = [word.translate(table) for word in text]\n",
        "\n",
        "        # remove tokens with numbers in them\n",
        "        text = [word for word in text if word.isalpha()]\n",
        "        \n",
        "        text1 = \" \".join(text)\n",
        "\n",
        "        return text, self.tokenizer.tokenize(text1)[0]\n",
        "\n",
        "    def build_vocabulary_and_dictionary(self, text_file_root, num_symbols):\n",
        "\n",
        "        self.dic = {}\n",
        "        \n",
        "        idx = 4\n",
        "        lines = [line.strip() for line in open(text_file_root)]\n",
        "\n",
        "        args = {\"mode\": \"aggressive\",\n",
        "                \"joiner_annotate\": True,\n",
        "                \"preserve_placeholders\": True,\n",
        "                \"case_markup\": True,\n",
        "                \"soft_case_regions\": True,\n",
        "                \"preserve_segmented_tokens\": True,}\n",
        "\n",
        "        tokenizer = pyonmttok.Tokenizer(**args)\n",
        "        learner = pyonmttok.BPELearner(tokenizer=tokenizer, symbols= num_symbols)\n",
        "        # learner = pyonmttok.SentencePieceLearner(vocab_size = num_symbols, character_coverage = 0.98)\n",
        "\n",
        "        for i in range(len(lines)):\n",
        "            learner.ingest(lines[i])\n",
        "\n",
        "        # learn and store bpe model\n",
        "        self.tokenizer = learner.learn(\"model.32k\")#(\"subword.bpe\")\n",
        "        \n",
        "        for i in range(len(lines)):\n",
        "\n",
        "            sentence, words =  self.preprocess(lines[i])\n",
        "            for word in words:\n",
        "                if word not in self.frequencies:\n",
        "                    self.frequencies[word] = 1\n",
        "\n",
        "                else:\n",
        "                    self.frequencies[word] += 1\n",
        "\n",
        "                if self.frequencies[word] == self.freq_threshold:\n",
        "                    self.str_2_idx[word] = idx\n",
        "                    self.idx_2_str[idx] = word\n",
        "                    idx += 1\n",
        "\n",
        "            self.dic[i] = [\" \".join(sentence)] \n",
        "\n",
        "    def IndexEncoding(self, text):\n",
        "        _, tokenized_text = self.preprocess(text)\n",
        "        index_encoded_text = [self.str_2_idx[token] if token in self.str_2_idx else self.str_2_idx[\"<unk>\"]\n",
        "                                for token in tokenized_text]\n",
        "        \n",
        "        return index_encoded_text\n",
        "\n",
        "    def IndexDecoding(self, sen_tensor):\n",
        "        tokenized_vec = list(sen_tensor.cpu().numpy())\n",
        "        decoded_vec = [self.idx_2_str[token] for token in tokenized_vec]\n",
        "        return decoded_vec\n",
        "\n",
        "    def get_str_2_idx(self):\n",
        "        return self.str_2_idx\n",
        "\n",
        "    def get_dic(self):\n",
        "        return self.dic"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fnDnC4P_-Efb",
        "outputId": "d58f349e-f86b-48e9-973f-e71f1dd355e6"
      },
      "source": [
        "en_vocab_dic_ = Vocabulary_and_Dictionary2(freq_threshold = 10)\n",
        "\n",
        "train_text_root = \"/content/AFEC-merged_en.txt\"\n",
        "en_vocab_dic_.build_vocabulary_and_dictionary(train_text_root, 32000)\n",
        "\n",
        "en_train_dic_ = en_vocab_dic_.get_dic() \n",
        "\n",
        "print(len(en_vocab_dic_.get_str_2_idx()))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "28778\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ueKTbA47-Efb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "026b3668-ff9b-441e-ade0-4dc7dd697997"
      },
      "source": [
        "en_train_dic_[0],len(en_train_dic_)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['north waziristan operation kills more militants'], 684417)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OXRP1Dmj-Efb",
        "outputId": "18dc8f11-348f-4eb5-e96b-b51ad1a784dd"
      },
      "source": [
        "fa_vocab_dic_ = Vocabulary_and_Dictionary2(freq_threshold = 10)\n",
        "\n",
        "train_text_root1 = \"/content/AFEC-merged_fa.txt\"\n",
        "fa_vocab_dic_.build_vocabulary_and_dictionary(train_text_root1,32000)\n",
        "\n",
        "fa_train_dic_ = fa_vocab_dic_.get_dic() \n",
        "\n",
        "print(len(fa_vocab_dic_.get_str_2_idx()))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "29012\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SoJ00dbd-Efc",
        "outputId": "f62efb37-90ae-4a40-de59-90a884464ff7"
      },
      "source": [
        "fa_train_dic_[0],len(fa_train_dic_)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['مرگ ستیزه جوی دیگر در عملیات وزیرستان شمالی'], 684417)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jed703us-Efc",
        "outputId": "c8d2beb9-ba7f-494f-95cd-879fc436df80"
      },
      "source": [
        "en_vocab_dic_2 = Vocabulary_and_Dictionary2(freq_threshold = 2)\n",
        "\n",
        "test_text_root = \"/content/test_en.txt\"\n",
        "en_vocab_dic_2.build_vocabulary_and_dictionary(test_text_root,357)\n",
        "\n",
        "en_test_dic_ = en_vocab_dic_2.get_dic() \n",
        "\n",
        "print(len(en_vocab_dic_2.get_str_2_idx()))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "285\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z0RnbgJV-Efc",
        "outputId": "82c9edba-70ec-496a-d719-0eb76a335c07"
      },
      "source": [
        "fa_vocab_dic_2 = Vocabulary_and_Dictionary2(freq_threshold = 2)\n",
        "\n",
        "test_text_root0 = \"/content/test_fa0.txt\"\n",
        "fa_vocab_dic_2.build_vocabulary_and_dictionary(test_text_root0,337)\n",
        "\n",
        "fa_test_dic_0 = fa_vocab_dic_2.get_dic() \n",
        "\n",
        "test_text_root1 = \"/content/test_fa1.txt\"\n",
        "fa_vocab_dic_2.build_vocabulary_and_dictionary(test_text_root1,337)\n",
        "\n",
        "fa_test_dic_1 = fa_vocab_dic_2.get_dic() \n",
        "\n",
        "test_text_root2 = \"/content/test_fa2.txt\"\n",
        "fa_vocab_dic_2.build_vocabulary_and_dictionary(test_text_root2,337)\n",
        "\n",
        "fa_test_dic_2 = fa_vocab_dic_2.get_dic() \n",
        "\n",
        "test_text_root3 = \"/content/test_fa3.txt\"\n",
        "fa_vocab_dic_2.build_vocabulary_and_dictionary(test_text_root3,337)\n",
        "\n",
        "fa_test_dic_3 = fa_vocab_dic_2.get_dic() \n",
        "\n",
        "print(len(fa_vocab_dic_2.get_str_2_idx()))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "460\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Pv_E7N3v-Efd",
        "outputId": "3f3e14fa-3c03-4809-dcc8-58925d3905dd"
      },
      "source": [
        "en_test_dic_"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{0: ['hello do we drive together to hanover on the twentyeighth of march'],\n",
              " 1: ['it is more comfortable by train'],\n",
              " 2: ['do you go by car and i go by train'],\n",
              " 3: ['i would like to go by train and what would you like'],\n",
              " 4: ['if we take the ice train at six past seven we will arrive at twentyfive past eight'],\n",
              " 5: ['which cafe'],\n",
              " 6: ['the cafe at platform fourteen'],\n",
              " 7: ['in any case a cheap hotel'],\n",
              " 8: ['what did you say please'],\n",
              " 9: ['and how much is a single room'],\n",
              " 10: ['we can take a taxi from the station to the hotel'],\n",
              " 11: ['at which hotel do you want to reserve a room now and how much is a single room'],\n",
              " 12: ['okay should we drive back on friday evening'],\n",
              " 13: ['i think we rather drive back at thirtythree past nine then we will arrive at hamburg at fiftytwo past ten'],\n",
              " 14: ['fine and dont forget your swimming stuff maybe we can go swimming together'],\n",
              " 15: ['yes when and where do we want to meet'],\n",
              " 16: ['i prefer the plane'],\n",
              " 17: ['a good idea then we will meet at the airport tomorrow'],\n",
              " 18: ['no idea we will see it does not matter'],\n",
              " 19: ['good let us meet at nine oclock hopefully the plane wont be hijacked tomorrow'],\n",
              " 20: ['i have already booked two rooms at the grunschnabel'],\n",
              " 21: ['what did you say'],\n",
              " 22: ['yes we have two rooms at the grunschnabel i will reserve a taxi right now'],\n",
              " 23: ['what is planned for the evening'],\n",
              " 24: ['a good idea i have heard phantom of the opera is supposed to be played'],\n",
              " 25: ['fine i think we have arranged everything then we will meet tomorrow'],\n",
              " 26: ['hello we have to talk about our trip to hanover'],\n",
              " 27: ['right we will be at the expo two thousand in hanover on the fourth and fifth of september'],\n",
              " 28: ['i have already booked a flight'],\n",
              " 29: ['we will set off at a quarter past eight and arrive at hanover at twelve oclock'],\n",
              " 30: ['we will have to meet at the airport at seven oclock'],\n",
              " 31: ['we arrive at hanover at twelve oclock midday'],\n",
              " 32: ['fine which hotel do you have in mind'],\n",
              " 33: ['the intercontinental is my favourite hotel in hanover'],\n",
              " 34: ['thank you how do we go back again to hamburg'],\n",
              " 35: ['we will go back by train on the fifth of september'],\n",
              " 36: ['the best thing is we meet at the train station at eight oclock'],\n",
              " 37: ['exactly the train leaves hanover at six minutes after eight'],\n",
              " 38: ['we leave hanover at eight oclock and arrive at hamburg at half past nine'],\n",
              " 39: ['goodbye'],\n",
              " 40: ['i would prefer to fly'],\n",
              " 41: ['yes there is a flight at a quarter past nine'],\n",
              " 42: ['the plane arrives in hanover at twentyfive past twelve'],\n",
              " 43: ['i did not understand what you said about the hotel'],\n",
              " 44: ['yes would you please book two rooms'],\n",
              " 45: ['the single room costs a hundred deutschmarks did i understand you right'],\n",
              " 46: ['where is this hotel'],\n",
              " 47: ['when do we meet'],\n",
              " 48: ['yes would you please book this hotel'],\n",
              " 49: ['please repeat this once again'],\n",
              " 50: ['do you mean you always stay there'],\n",
              " 51: ['fine would you like to do something in the evening'],\n",
              " 52: ['unfortunately i did not understand you would you like to go to the cinema'],\n",
              " 53: ['would you like to go out for a meal afterwards'],\n",
              " 54: ['we still have to plan our return trip'],\n",
              " 55: ['when do we want to travel back to hamburg'],\n",
              " 56: ['could you repeat that once again please'],\n",
              " 57: ['yes i agree should we also fly back or go by train'],\n",
              " 58: ['there is a flight leaving hanover at a quarter past seven pm'],\n",
              " 59: ['fine shall i book this flight'],\n",
              " 60: ['goodbye'],\n",
              " 61: ['no we go on the twentythird'],\n",
              " 62: ['no i suggest to take the train in the morning'],\n",
              " 63: ['six oclock is too early i propose to take the train at ten oclock in the morning'],\n",
              " 64: ['do you agree'],\n",
              " 65: ['i did not understand that'],\n",
              " 66: ['i did not understand that again'],\n",
              " 67: ['i still have not understood anything'],\n",
              " 68: ['i suggest to stay at the hotel grunschnabel in hanover'],\n",
              " 69: ['yes please'],\n",
              " 70: ['so shall we take the hotel grunschnabel'],\n",
              " 71: ['well once again should we stay at the hotel grunschnabel'],\n",
              " 72: ['once again shall we stay at the hotel grunschnabel'],\n",
              " 73: ['yes that is a good idea'],\n",
              " 74: ['i did not understand that i suggest to go back to hamburg on the twentyfourth in the evening'],\n",
              " 75: ['yes i agree i propose to take the train again'],\n",
              " 76: ['i agree everything okay'],\n",
              " 77: ['hello should we plan our journey to the expo on october the ninth'],\n",
              " 78: ['i would prefer to take the train'],\n",
              " 79: ['we could leave hamburg at twentyfour past seven and we would be in hanover at thirtynine past eight'],\n",
              " 80: ['fine and where at the station'],\n",
              " 81: ['do you also want to reserve seats for us'],\n",
              " 82: ['are the single rooms cheap there'],\n",
              " 83: ['three hundred deutschmarks is a bit expensive isnt there anything cheaper'],\n",
              " 84: ['are there single rooms for eighty deutschmarks'],\n",
              " 85: ['fine but at which hotel do you want to reserve now'],\n",
              " 86: ['where is our hotel and what is its name'],\n",
              " 87: ['at which hotel do you want to reserve for us and where is the hotel'],\n",
              " 88: ['would you repeat the hotel name then please'],\n",
              " 89: ['what is the name of our hotel'],\n",
              " 90: ['in the evening i would like to relax and maybe go to the sauna what do you think'],\n",
              " 91: ['okay the program of the evening we can plan later on'],\n",
              " 92: ['i would like to take the train at eighteen past nine pm is that okay for you'],\n",
              " 93: ['fine see you later then'],\n",
              " 94: ['fine at what time would you like to leave'],\n",
              " 95: ['i would prefer the train'],\n",
              " 96: ['that is okay should we meet at the station at eight oclock then'],\n",
              " 97: ['okay do you want me to reserve a hotel for us'],\n",
              " 98: ['i would suggest the stadthotel because it is the cheapest one'],\n",
              " 99: ['the stadthotel'],\n",
              " 100: ['would you like a single room or a double room'],\n",
              " 101: ['okay should we go to the hotel by taxi immediately'],\n",
              " 102: ['would you like to do something in the evening'],\n",
              " 103: ['fine i agree with that'],\n",
              " 104: ['could you find out a train connection for wednesday evening'],\n",
              " 105: ['sorry when will we arrive at hamburg'],\n",
              " 106: ['that is okay then we can go back by taxi'],\n",
              " 107: ['okay goodbye'],\n",
              " 108: ['hello i call you to plan the journey to hanover with you'],\n",
              " 109: ['yes i would like to leave on wednesday the twentysixth of january'],\n",
              " 110: ['i prefer the plane'],\n",
              " 111: ['fine at what time should we leave'],\n",
              " 112: ['i propose to leave at nine oclock in the morning'],\n",
              " 113: ['where should we meet'],\n",
              " 114: ['at which office please'],\n",
              " 115: ['fine do you want me to reserve a hotel for us'],\n",
              " 116: ['the hotel is called prinzenhof and it is in the city center'],\n",
              " 117: ['do you agree on a double room'],\n",
              " 118: ['great i will reserve two single rooms'],\n",
              " 119: ['would you like to go out for a meal in the evening'],\n",
              " 120: ['fine i will choose a restaurant'],\n",
              " 121: ['i would like to leave on thursday afternoon'],\n",
              " 122: ['yes i agree with that'],\n",
              " 123: ['yes i agree that we leave at five oclock'],\n",
              " 124: ['i think we have talked about everything goodbye'],\n",
              " 125: ['should we meet at the expo in hanover in june'],\n",
              " 126: ['yes i thought we would go to hanover on the ninth and tenth of june'],\n",
              " 127: ['yes i have got a timetable when would you like to go'],\n",
              " 128: ['ahm there is a train leaving hamburg at twentyfour past ten it arrives at hanover at thirtynine past eleven'],\n",
              " 129: ['i agree then we could have breakfast together'],\n",
              " 130: ['i agree we need a hotel in hanover'],\n",
              " 131: ['unfortunately i did not understand you'],\n",
              " 132: ['in may on the eighteenth and nineteenth of may we are going together to hanover'],\n",
              " 133: ['i would like to ahm go by plane do you have a plan'],\n",
              " 134: ['i would like to fly and to be in hanover at about ten oclock'],\n",
              " 135: ['where should we meet'],\n",
              " 136: ['should we meet at the meeting point at the airport'],\n",
              " 137: ['would you please repeat that once again'],\n",
              " 138: ['i agree how can i recognize you'],\n",
              " 139: ['well when would you like to fly back'],\n",
              " 140: ['i agree when exactly does the plane take off'],\n",
              " 141: ['please repeat that once again'],\n",
              " 142: ['that sounds great ahm fine you arrange the flight'],\n",
              " 143: ['do you want me to ah book the hotel do you have any preferences'],\n",
              " 144: ['a single room and it is important for you to have a bar'],\n",
              " 145: ['i think there is a tv in all the hotels'],\n",
              " 146: ['there is a hotel near the fairground but it is very expensive'],\n",
              " 147: ['i agree a single room costs three hundred deutschmarks a night'],\n",
              " 148: ['okay i book the hotel'],\n",
              " 149: ['ahm should we meet on may the eighteenth in the evening'],\n",
              " 150: ['okay should we meet at the hotel at half past seven'],\n",
              " 151: ['fine we still have to arrange the ride from the airport to the hotel'],\n",
              " 152: ['i did not understand that'],\n",
              " 153: ['i suggest we take the subway to the hotel'],\n",
              " 154: ['the subway takes about twenty minutes to the station'],\n",
              " 155: ['i agree then we have to have dinner earlier'],\n",
              " 156: ['yes is everything okay so far then'],\n",
              " 157: ['yes well see you on the eighteenth then'],\n",
              " 158: ['hello i would like to go with you to the expo in hanover'],\n",
              " 159: ['i did not understand that would you prefer to fly or to drive'],\n",
              " 160: ['i would like to be in hanover until noon but i still do not know if you want to fly or to drive'],\n",
              " 161: ['that sounds very good where should we meet'],\n",
              " 162: ['yes fine how would half past seven suit you'],\n",
              " 163: ['fine i have a list of hotels for hanover'],\n",
              " 164: ['i would prefer a hotel in the city center then we can do something in the evening'],\n",
              " 165: ['unfortunately i did not understand that'],\n",
              " 166: ['i could not understand you again'],\n",
              " 167: ['fine then i propose the hotel prinzenhof that is situated in the city center'],\n",
              " 168: ['did you understand that i have proposed the hotel prinzenhof'],\n",
              " 169: ['fine i will book two single rooms for two hundred deutschmarks each'],\n",
              " 170: ['do you know a restaurant in hanover for i do not know any'],\n",
              " 171: ['i would really like to have chinese or japanese food but not italien food is that possible'],\n",
              " 172: ['that is fine then we still have to arrange the return journey do you have a schedule for a return flight'],\n",
              " 173: ['i have to be back in hamburg on friday at three oclock pm at the latest'],\n",
              " 174: ['i did not understand when the plane leaves hanover and when it is supposed to arrive at hamburg'],\n",
              " 175: ['i did not understand that yet'],\n",
              " 176: ['that is fine when does the plane leave hanover'],\n",
              " 177: ['then we arrive at the airport in hamburg again right'],\n",
              " 178: ['then everything is sorted out now then we will see each other on august the seventeenth'],\n",
              " 179: ['okay goodbye'],\n",
              " 180: ['good morning how are you'],\n",
              " 181: ['when should we go to hanover'],\n",
              " 182: ['fine let us meet at ten oclock in the morning should we go by train'],\n",
              " 183: ['thanks then we meet at the train station'],\n",
              " 184: ['when will we arrive in hanover'],\n",
              " 185: ['i have asked when we will arrive in hanover'],\n",
              " 186: ['fine i have already reserved a hotel'],\n",
              " 187: ['at the hotel maritim we can go there by taxi'],\n",
              " 188: ['yes i have already booked the hotel'],\n",
              " 189: ['i have booked two single rooms one for me and one for you'],\n",
              " 190: ['in the evening we should watch the newest movie with arnold schwarzenegger'],\n",
              " 191: ['fine let us meet at the station at nine oclock on thursday morning'],\n",
              " 192: ['what did you say please'],\n",
              " 193: ['yes we leave at nine oclock and arrive at hamburg at one oclock right'],\n",
              " 194: ['yes is everything cleared up then'],\n",
              " 195: ['then everything is arranged goodbye'],\n",
              " 196: ['hello i would prefer the intercontinental'],\n",
              " 197: ['fine let us take the intercontinental should we fly'],\n",
              " 198: ['no problem then let us meet where should we meet'],\n",
              " 199: ['yes we take the atlantichotel'],\n",
              " 200: ['thanks by the way we fly to hanover'],\n",
              " 201: ['yes our plane will take off at a quarter to seven and will land at hanover at ten past ten'],\n",
              " 202: ['fine by taxi we will be at the hotel within ten minutes'],\n",
              " 203: ['yes let us take two single rooms and what would you like to do in the evening'],\n",
              " 204: ['fine let us go out for a meal in the evening the best would be the grunschnabel'],\n",
              " 205: ['on wednesday we will go back by train to hamburg'],\n",
              " 206: ['we will leave at half past eight and will arrive at hamburg at ten oclock'],\n",
              " 207: ['the best thing would be for us to take the subway from our hotel to the station'],\n",
              " 208: ['fine let us go for a little walk in hamburg'],\n",
              " 209: ['i am looking forward to that goodbye'],\n",
              " 210: ['hello should we go by car to hanover'],\n",
              " 211: ['we should leave at ten oclock in the morning to be in hanover at twelve oclock'],\n",
              " 212: ['do you want me to book a hotel'],\n",
              " 213: ['i will book two single rooms with a shower at the konigshotel'],\n",
              " 214: ['yes i would suggest we fly back'],\n",
              " 215: ['yes i would suggest the flight at a quarter past seven'],\n",
              " 216: ['i agree'],\n",
              " 217: ['we will be back in hamburg at five past ten'],\n",
              " 218: ['yes we will be back at hamburg at five past ten would you like to go to the wienerwald in hanover in the evening'],\n",
              " 219: ['okay goodbye'],\n",
              " 220: ['mrs schmidt we still want to plan the journey to hanover'],\n",
              " 221: ['i would like to take the train on the thirtieth'],\n",
              " 222: ['during a flight i always get sick i would like to take the train'],\n",
              " 223: ['well i would like to take the train during a flight i get sick'],\n",
              " 224: ['please let us take the train'],\n",
              " 225: ['let us take the train at nine oclock'],\n",
              " 226: ['at six past nine then then let us meet at the station at half past eight'],\n",
              " 227: ['when do we arrive at hanover'],\n",
              " 228: ['that is very good at which hotel should we stay then what about the grunschnabel'],\n",
              " 229: ['i think we will take the metropol hotel could you reserve two single rooms'],\n",
              " 230: ['that is perfect would you like to go out for a meal in the evening'],\n",
              " 231: ['yes the steaks at the blockhouse are always very succulent'],\n",
              " 232: ['good idea we could take the one at the maschsee'],\n",
              " 233: ['that is perfect then we have talked about everything goodbye'],\n",
              " 234: ['hello we still have to plan our trip to hanover'],\n",
              " 235: ['i would like to fly that looks more important'],\n",
              " 236: ['the earliest flight leaves at five past eight should we take that one'],\n",
              " 237: ['the plane arrives at hanover at a quarter past eleven'],\n",
              " 238: ['let us meet at seven oclock we still have to check in'],\n",
              " 239: ['what do you think about the hotel gewandhaus'],\n",
              " 240: ['i would like one i would like to have one single room'],\n",
              " 241: ['you would like to have a room with tv and bar'],\n",
              " 242: ['i will book the hotelroom'],\n",
              " 243: ['we have already agreed on the hotel'],\n",
              " 244: ['unfortunately i did not understand that'],\n",
              " 245: ['let us leave at eight oclock pm there is a flight to hamburg going over milan isnt that great'],\n",
              " 246: ['let yourself get surprised'],\n",
              " 247: ['i could but i dont do it'],\n",
              " 248: ['at eight oclock pm'],\n",
              " 249: ['yes that is in the evening now we have talked about everything goodbye']}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H9-0nQDl-Efd",
        "outputId": "f8138846-9692-4859-81b8-c88113abe955"
      },
      "source": [
        "fa_vocab_dic_2.get_str_2_idx()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'<eos>': 2,\n",
              " '<pad>': 0,\n",
              " '<sos>': 1,\n",
              " '<unk>': 3,\n",
              " 'آلمان': 174,\n",
              " 'آلمانی': 33,\n",
              " 'آن': 143,\n",
              " 'آنجا': 169,\n",
              " 'آن￭': 276,\n",
              " 'آیا': 7,\n",
              " 'آ￭': 6,\n",
              " 'ئ￭': 226,\n",
              " 'ا': 107,\n",
              " 'اتاق': 26,\n",
              " 'اتا￭': 288,\n",
              " 'ات￭': 4,\n",
              " 'اجازه': 269,\n",
              " 'اج￭': 14,\n",
              " 'اح￭': 37,\n",
              " 'ار': 13,\n",
              " 'ارز￭': 170,\n",
              " 'اری': 39,\n",
              " 'ار￭': 38,\n",
              " 'از': 137,\n",
              " 'از￭': 82,\n",
              " 'است': 6,\n",
              " 'اس￭': 35,\n",
              " 'اقامت': 29,\n",
              " 'الان': 11,\n",
              " 'الی': 64,\n",
              " 'ام': 239,\n",
              " 'اما': 65,\n",
              " 'ام￭': 24,\n",
              " 'ان': 171,\n",
              " 'انت￭': 216,\n",
              " 'انجا': 30,\n",
              " 'انجام': 202,\n",
              " 'انه': 231,\n",
              " 'انو￭': 41,\n",
              " 'ان￭': 177,\n",
              " 'اه': 43,\n",
              " 'اهم': 46,\n",
              " 'اوم': 58,\n",
              " 'اکسپو': 164,\n",
              " 'اکسپ￭': 16,\n",
              " 'اک￭': 17,\n",
              " 'ای': 257,\n",
              " 'ایا': 4,\n",
              " 'اید': 35,\n",
              " 'ایده': 9,\n",
              " 'ایستگاه': 106,\n",
              " 'ایست￭': 26,\n",
              " 'ایم': 84,\n",
              " 'این': 128,\n",
              " 'اینجا': 148,\n",
              " 'این￭': 95,\n",
              " 'ای￭': 38,\n",
              " 'ا￭': 39,\n",
              " 'ب': 217,\n",
              " 'با': 4,\n",
              " 'بار': 240,\n",
              " 'باز￭': 139,\n",
              " 'باشد': 36,\n",
              " 'باشم': 249,\n",
              " 'باشه': 189,\n",
              " 'باشیم': 28,\n",
              " 'باش￭': 241,\n",
              " 'باهم': 8,\n",
              " 'باید': 45,\n",
              " 'با￭': 205,\n",
              " 'ببینیم': 55,\n",
              " 'بده': 60,\n",
              " 'بدهید': 78,\n",
              " 'بدهیم': 38,\n",
              " 'بده￭': 19,\n",
              " 'برای': 72,\n",
              " 'برنامه': 76,\n",
              " 'بروم': 188,\n",
              " 'بروی': 138,\n",
              " 'بروید': 26,\n",
              " 'برویم': 201,\n",
              " 'برو￭': 20,\n",
              " 'برگردیم': 34,\n",
              " 'برگر￭': 61,\n",
              " 'برگ￭': 73,\n",
              " 'بر￭': 51,\n",
              " 'بسیار': 19,\n",
              " 'بسی￭': 42,\n",
              " 'بعد': 6,\n",
              " 'بعدازظهر': 12,\n",
              " 'بعداز￭': 6,\n",
              " 'بع￭': 191,\n",
              " 'بله': 68,\n",
              " 'بل￭': 274,\n",
              " 'بمانیم': 160,\n",
              " 'به': 59,\n",
              " 'بهت￭': 104,\n",
              " 'بود': 275,\n",
              " 'بگیرم': 62,\n",
              " 'بگیریم': 155,\n",
              " 'بگیر￭': 24,\n",
              " 'بی': 30,\n",
              " 'بیا': 27,\n",
              " 'بیرون': 215,\n",
              " 'بیست': 13,\n",
              " 'بینیم': 56,\n",
              " 'بین￭': 49,\n",
              " 'بی￭': 40,\n",
              " 'ب￭': 41,\n",
              " 'ت': 186,\n",
              " 'تا': 23,\n",
              " 'تاکسی': 70,\n",
              " 'تا￭': 96,\n",
              " 'تر': 67,\n",
              " 'ترتیب': 64,\n",
              " 'ترجیح': 111,\n",
              " 'ترک': 109,\n",
              " 'ترک￭': 49,\n",
              " 'تر￭': 178,\n",
              " 'تش￭': 235,\n",
              " 'ته': 69,\n",
              " 'تو': 5,\n",
              " 'توا￭': 203,\n",
              " 'تکرار': 144,\n",
              " 'تکنفره': 180,\n",
              " 'ت￭': 5,\n",
              " 'ث￭': 74,\n",
              " 'جا': 11,\n",
              " 'ج￭': 119,\n",
              " 'حاضر': 87,\n",
              " 'حافظ': 85,\n",
              " 'حال': 86,\n",
              " 'حالا': 181,\n",
              " 'حال￭': 279,\n",
              " 'حا￭': 55,\n",
              " 'حرف': 133,\n",
              " 'حرکت': 221,\n",
              " 'ح￭': 185,\n",
              " 'خب': 14,\n",
              " 'خداحافظ': 147,\n",
              " 'خدانگهدار': 13,\n",
              " 'خواهد': 117,\n",
              " 'خواهم': 218,\n",
              " 'خواهی': 23,\n",
              " 'خواهیم': 85,\n",
              " 'خواه￭': 18,\n",
              " 'خوا￭': 48,\n",
              " 'خوب': 49,\n",
              " 'خوبی': 13,\n",
              " 'خو￭': 194,\n",
              " 'خیلی': 151,\n",
              " 'خی￭': 267,\n",
              " 'خ￭': 132,\n",
              " 'د': 64,\n",
              " 'دا': 61,\n",
              " 'دارد': 126,\n",
              " 'دارم': 56,\n",
              " 'داری': 135,\n",
              " 'دارید': 94,\n",
              " 'داریم': 8,\n",
              " 'دار￭': 230,\n",
              " 'داشته': 287,\n",
              " 'دا￭': 219,\n",
              " 'دت': 63,\n",
              " 'در': 21,\n",
              " 'درباره': 16,\n",
              " 'درساعت': 4,\n",
              " 'درست': 20,\n",
              " 'درمورد': 20,\n",
              " 'در￭': 266,\n",
              " 'دقیقه': 14,\n",
              " 'دقیق￭': 236,\n",
              " 'دم': 197,\n",
              " 'ده': 36,\n",
              " 'دهم': 233,\n",
              " 'دهید': 270,\n",
              " 'ده￭': 247,\n",
              " 'دو': 65,\n",
              " 'دوازده': 92,\n",
              " 'دوا￭': 52,\n",
              " 'دوباره': 129,\n",
              " 'دوست': 11,\n",
              " 'دوشزمارک': 18,\n",
              " 'دو￭': 199,\n",
              " 'دید': 62,\n",
              " 'دیم': 224,\n",
              " 'دیگر': 159,\n",
              " 'دی￭': 59,\n",
              " 'د￭': 195,\n",
              " 'ذ￭': 16,\n",
              " 'ر': 32,\n",
              " 'را': 46,\n",
              " 'را￭': 118,\n",
              " 'ربع': 114,\n",
              " 'رب￭': 9,\n",
              " 'رزرو': 66,\n",
              " 'رز￭': 58,\n",
              " 'رستور￭': 7,\n",
              " 'رسید': 89,\n",
              " 'رس￭': 51,\n",
              " 'رو': 51,\n",
              " 'روز': 100,\n",
              " 'رو￭': 74,\n",
              " 'رک￭': 52,\n",
              " 'ری': 131,\n",
              " 'ریزی': 165,\n",
              " 'ریم': 245,\n",
              " 'ری￭': 79,\n",
              " 'ر￭': 204,\n",
              " 'ز': 212,\n",
              " 'زده': 289,\n",
              " 'زمانی': 193,\n",
              " 'زم￭': 25,\n",
              " 'زی': 32,\n",
              " 'ز￭': 124,\n",
              " 'س': 286,\n",
              " 'ساعت': 12,\n",
              " 'ساع￭': 40,\n",
              " 'سا￭': 61,\n",
              " 'ست': 115,\n",
              " 'ست￭': 172,\n",
              " 'سفر': 207,\n",
              " 'سفرمان': 27,\n",
              " 'سف￭': 254,\n",
              " 'سلام': 79,\n",
              " 'سه': 66,\n",
              " 'سوار': 149,\n",
              " 'سپس': 22,\n",
              " 'سپ￭': 102,\n",
              " 'سی': 166,\n",
              " 'سی￭': 11,\n",
              " 'س￭': 33,\n",
              " 'ش': 277,\n",
              " 'شا￭': 187,\n",
              " 'شت': 264,\n",
              " 'شته': 75,\n",
              " 'شت￭': 140,\n",
              " 'شده': 76,\n",
              " 'شش': 108,\n",
              " 'شما': 17,\n",
              " 'شنبه': 208,\n",
              " 'شن￭': 43,\n",
              " 'شوم': 42,\n",
              " 'شویم': 150,\n",
              " 'شو￭': 290,\n",
              " 'ش￭': 63,\n",
              " 'ص': 47,\n",
              " 'صبح': 154,\n",
              " 'صب￭': 54,\n",
              " 'صحبت': 223,\n",
              " 'ص￭': 167,\n",
              " 'ض': 81,\n",
              " 'طور': 77,\n",
              " 'ط￭': 268,\n",
              " 'ظهر': 70,\n",
              " 'ظ￭': 220,\n",
              " 'ع': 222,\n",
              " 'عازم': 47,\n",
              " 'عالی': 281,\n",
              " 'عصر': 73,\n",
              " 'عص￭': 37,\n",
              " 'ع￭': 261,\n",
              " 'غذا': 214,\n",
              " 'غذ￭': 50,\n",
              " 'ف': 256,\n",
              " 'فت': 23,\n",
              " 'فت￭': 210,\n",
              " 'فره': 83,\n",
              " 'فرودگاه': 91,\n",
              " 'فر￭': 52,\n",
              " 'فهمیدم': 17,\n",
              " 'فکر': 48,\n",
              " 'ف￭': 200,\n",
              " 'ق': 60,\n",
              " 'قا￭': 25,\n",
              " 'قبلا': 14,\n",
              " 'قرار': 75,\n",
              " 'قطار': 9,\n",
              " 'قط￭': 53,\n",
              " 'قیمت': 10,\n",
              " 'قی￭': 27,\n",
              " 'ق￭': 31,\n",
              " 'ل': 97,\n",
              " 'لان': 65,\n",
              " 'لا￭': 282,\n",
              " 'لطفا': 122,\n",
              " 'لی': 243,\n",
              " 'ل￭': 242,\n",
              " 'م': 83,\n",
              " 'ما': 22,\n",
              " 'مارک': 173,\n",
              " 'مار￭': 271,\n",
              " 'مان': 141,\n",
              " 'مانی': 48,\n",
              " 'مان￭': 18,\n",
              " 'مایلم': 36,\n",
              " 'مایلید': 24,\n",
              " 'ما￭': 206,\n",
              " 'مب￭': 103,\n",
              " 'مت': 28,\n",
              " 'متاسفانه': 44,\n",
              " 'متاسف￭': 46,\n",
              " 'متر￭': 244,\n",
              " 'متوجه': 21,\n",
              " 'مت￭': 7,\n",
              " 'مجبوریم': 12,\n",
              " 'مرتب': 246,\n",
              " 'مر￭': 253,\n",
              " 'ملاقات': 54,\n",
              " 'ممکن': 263,\n",
              " 'من': 10,\n",
              " 'منظور': 22,\n",
              " 'منظورت': 10,\n",
              " 'موافقم': 161,\n",
              " 'موافق￭': 213,\n",
              " 'مورد': 80,\n",
              " 'می': 57,\n",
              " 'میبی￭': 78,\n",
              " 'میتوانی': 34,\n",
              " 'میتوانیم': 190,\n",
              " 'میتوان￭': 229,\n",
              " 'میخواهی': 32,\n",
              " 'میخواهید': 182,\n",
              " 'میخواه￭': 250,\n",
              " 'میدهم': 112,\n",
              " 'مید￭': 50,\n",
              " 'میرسد': 63,\n",
              " 'میرسیم': 265,\n",
              " 'میرس￭': 285,\n",
              " 'میشود': 183,\n",
              " 'میش￭': 259,\n",
              " 'میکند': 145,\n",
              " 'میکنم': 71,\n",
              " 'میکنیم': 127,\n",
              " 'میکن￭': 15,\n",
              " 'میگیر￭': 39,\n",
              " 'می￭': 60,\n",
              " 'م￭': 40,\n",
              " 'ن': 121,\n",
              " 'نام': 184,\n",
              " 'نا￭': 50,\n",
              " 'نجا': 68,\n",
              " 'ند': 237,\n",
              " 'ند￭': 168,\n",
              " 'نشدم': 23,\n",
              " 'نش￭': 44,\n",
              " 'نظر': 238,\n",
              " 'نظ￭': 130,\n",
              " 'نفره': 29,\n",
              " 'نفهمیدم': 134,\n",
              " 'نمایشگاه': 31,\n",
              " 'نه': 58,\n",
              " 'نها￭': 71,\n",
              " 'نو￭': 234,\n",
              " 'نی': 72,\n",
              " 'نیست': 179,\n",
              " 'نیم': 57,\n",
              " 'نی￭': 56,\n",
              " 'ن￭': 55,\n",
              " 'ه': 15,\n",
              " 'ها': 251,\n",
              " 'هامبورگ': 99,\n",
              " 'هانوفر': 15,\n",
              " 'هانوور': 81,\n",
              " 'هانو￭': 28,\n",
              " 'های': 31,\n",
              " 'ها￭': 211,\n",
              " 'هتل': 24,\n",
              " 'هت￭': 252,\n",
              " 'هج￭': 232,\n",
              " 'هر': 93,\n",
              " 'هزار': 19,\n",
              " 'هشت': 88,\n",
              " 'هفت': 90,\n",
              " 'هم': 42,\n",
              " 'همدیگر': 12,\n",
              " 'همه': 162,\n",
              " 'همی￭': 278,\n",
              " 'هم￭': 45,\n",
              " 'هنوز': 156,\n",
              " 'هواپیما': 62,\n",
              " 'هواپی￭': 59,\n",
              " 'هی￭': 21,\n",
              " 'ه￭': 84,\n",
              " 'و': 8,\n",
              " 'وا￭': 260,\n",
              " 'وجود': 22,\n",
              " 'وج￭': 27,\n",
              " 'ود': 152,\n",
              " 'ود￭': 284,\n",
              " 'ور': 53,\n",
              " 'ور￭': 258,\n",
              " 'وقت': 227,\n",
              " 'وم': 5,\n",
              " 'ون': 43,\n",
              " 'وید': 41,\n",
              " 'ویم': 28,\n",
              " 'و￭': 37,\n",
              " 'پرواز': 113,\n",
              " 'پر￭': 255,\n",
              " 'پس': 77,\n",
              " 'پنج': 116,\n",
              " 'پنج￭': 101,\n",
              " 'پن￭': 262,\n",
              " 'پیشنهاد': 153,\n",
              " 'پی￭': 45,\n",
              " 'پ￭': 280,\n",
              " 'چ': 176,\n",
              " 'چه': 18,\n",
              " 'چهار￭': 82,\n",
              " 'چها￭': 228,\n",
              " 'چی': 8,\n",
              " 'چیز': 163,\n",
              " 'چی￭': 157,\n",
              " 'چ￭': 30,\n",
              " 'ژ￭': 225,\n",
              " 'ک': 283,\n",
              " 'کا￭': 120,\n",
              " 'کجا': 209,\n",
              " 'کج￭': 34,\n",
              " 'کدام': 25,\n",
              " 'کرد': 110,\n",
              " 'کر￭': 196,\n",
              " 'کن': 5,\n",
              " 'کنفره': 9,\n",
              " 'کنم': 146,\n",
              " 'کنی': 29,\n",
              " 'کنید': 123,\n",
              " 'کنیم': 47,\n",
              " 'کن￭': 248,\n",
              " 'که': 105,\n",
              " 'کی': 25,\n",
              " 'ک￭': 44,\n",
              " 'گ': 15,\n",
              " 'گذ￭': 54,\n",
              " 'گر': 273,\n",
              " 'گرانشابل': 69,\n",
              " 'گرانشنبل': 7,\n",
              " 'گردیم': 21,\n",
              " 'گرد￭': 26,\n",
              " 'گریچنبل': 10,\n",
              " 'گر￭': 175,\n",
              " 'گشت': 17,\n",
              " 'گفت￭': 67,\n",
              " 'گ￭': 98,\n",
              " 'ی': 16,\n",
              " 'یا': 198,\n",
              " 'ید': 19,\n",
              " 'یر￭': 80,\n",
              " 'یز￭': 192,\n",
              " 'یست': 33,\n",
              " 'یم': 142,\n",
              " 'ین': 35,\n",
              " 'ین￭': 136,\n",
              " 'یک': 20,\n",
              " 'یکبار': 158,\n",
              " 'یکدیگر': 53,\n",
              " 'یکی': 57,\n",
              " 'یک￭': 272,\n",
              " 'ی￭': 125}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UTKc3rSCCpGa"
      },
      "source": [
        "### AFEC Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8cLmI1R4CpGb"
      },
      "source": [
        "class AFEC_Dataset(torch.utils.data.Dataset):\n",
        "    \"\"\"\n",
        "    Characterize AFEC Dataset for PyTorch. To be compatible with \n",
        "    DataLoader and other PyTorch syntaxes.\n",
        "    \"\"\"\n",
        "    def __init__(self, src_dic, trg_dic, indices, src_vocab, trg_vocab):\n",
        "        \n",
        "        self.src_dic = src_dic\n",
        "        self.trg_dic = trg_dic\n",
        "\n",
        "        # Get img names, caption columns\n",
        "        self.src_sentences = [self.src_dic[i] for i in indices]\n",
        "        self.trg_sentences = [self.trg_dic[i] for i in indices]\n",
        "\n",
        "        self.src_vocab = src_vocab\n",
        "        self.trg_vocab = trg_vocab \n",
        "            \n",
        "    def __len__(self):\n",
        "        'Denotes the total number of samples'\n",
        "        return len(self.src_sentences)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        \"\"\" \n",
        "        Generates one sample of data with label\n",
        "        \n",
        "        Arguments\n",
        "        ------------\n",
        "        index : integer, in range of len(self.list_IDs)\n",
        "                index of sample in list of IDs,\n",
        "        \n",
        "        Returns\n",
        "        ------------\n",
        "        X : torch tensor,\n",
        "            input data\n",
        "\n",
        "        Y : torch tensor,\n",
        "            correspond label of input data\n",
        "        \"\"\"\n",
        "        # Select sample\n",
        "        src_sentence = [ self.src_vocab.str_2_idx[\"<sos>\"] ]\n",
        "        src_sentence += self.src_vocab.IndexEncoding(self.src_sentences[index][0])\n",
        "        src_sentence.append(self.src_vocab.str_2_idx[\"<eos>\"])\n",
        "\n",
        "        trg_sentence = [ self.trg_vocab.str_2_idx[\"<sos>\"] ]\n",
        "        trg_sentence += self.trg_vocab.IndexEncoding(self.trg_sentences[index][0])\n",
        "        trg_sentence.append(self.trg_vocab.str_2_idx[\"<eos>\"])\n",
        "       \n",
        "        tensor_src_sentence = torch.tensor(src_sentence)\n",
        "        tensor_trg_sentence = torch.tensor(trg_sentence)\n",
        "\n",
        "        return tensor_src_sentence , tensor_trg_sentence"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M8LWwWMtCpGb"
      },
      "source": [
        "### Split Dataset and Activate CUDA"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3L_pfrW7CpGb"
      },
      "source": [
        "#train/val/test split \n",
        "random.seed(2045)\n",
        "\n",
        "indices = list(range(len(en_train_dic_)))\n",
        "num_samples = len(en_train_dic_)\n",
        "random.shuffle(indices)\n",
        "\n",
        "train_indices = indices[0 : int(0.9 * num_samples)]\n",
        "validation_indices = indices[int(0.9 * num_samples) : ]\n",
        "test_indices = list(range(len(en_test_dic_)))\n",
        "\n",
        "partition2 = {'train' : train_indices , 'validation' : validation_indices,\n",
        "             'test' : test_indices }"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tg-fel-dCpGd"
      },
      "source": [
        "# CUDA for PyTorch\n",
        "use_cuda = torch.cuda.is_available()\n",
        "device = torch.device(\"cuda:0\" if use_cuda else \"cpu\")\n",
        "torch.backends.cudnn.benchmark = True"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "azpf7SFcCpGd",
        "outputId": "db8e1288-f9e8-4888-e210-c33de0f7eed5"
      },
      "source": [
        "device"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "device(type='cuda', index=0)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pwWWtXjFCpGd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bdcb4d3d-5679-4234-f943-fc20d5db15ab"
      },
      "source": [
        "len(partition2['test'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "250"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kkiJd3mxCpGe"
      },
      "source": [
        "### Padding"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZY6h_AEDCpGe"
      },
      "source": [
        "class BatchPadCollate:\n",
        "    def __init__(self, pad_idx):\n",
        "        self.pad_idx = pad_idx\n",
        "\n",
        "    def __call__(self, batch):\n",
        "\n",
        "        src_sentences = [item[0] for item in batch]\n",
        "        src_sentences = pad_sequence(src_sentences, batch_first= True, padding_value=self.pad_idx)\n",
        "\n",
        "        trg_sentences = [item[1] for item in batch]\n",
        "        trg_sentences = pad_sequence(trg_sentences, batch_first= True, padding_value=self.pad_idx) \n",
        "        \n",
        "        return src_sentences, trg_sentences "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xjkPg1R7VqGZ"
      },
      "source": [
        "## Transformer Network "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OhKmrW9CVqGZ"
      },
      "source": [
        "class PositionalEncoding(nn.Module):\n",
        "    \n",
        "    def __init__(self,embed_dim, p_dropout, maximum_position_encoding):\n",
        "        super(PositionalEncoding, self).__init__()\n",
        "        \n",
        "        self.max_pos = maximum_position_encoding\n",
        "        self.dropout = nn.Dropout(p_dropout)\n",
        "        pos_enc = torch.zeros(self.max_pos, embed_dim)\n",
        "        positions = torch.arange(0, self.max_pos, dtype=torch.float).unsqueeze(1)\n",
        "        div_term = torch.exp(torch.arange(0, embed_dim, 2).float() * (-math.log(10000.0) / embed_dim))\n",
        "        pos_enc[:, 0::2] = torch.sin(positions * div_term)\n",
        "        pos_enc[:, 1::2] = torch.cos(positions * div_term)\n",
        "        pos_enc = pos_enc.unsqueeze(0).transpose(0, 1)\n",
        "        self.register_buffer('pos_enc', pos_enc)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = x + self.pos_enc[:x.size(0), :]\n",
        "        x = self.dropout(x)\n",
        "        \n",
        "        return x\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RbWnxY8nVqGa"
      },
      "source": [
        "class _ScaledDotProductAttention(nn.Module):\n",
        "    \n",
        "    def __init__(self, embed_dim, device, p_dropout=0., kdim=None, vdim=None):\n",
        "        super(_ScaledDotProductAttention, self).__init__()\n",
        "        self.device = device\n",
        "        self.embed_dim = embed_dim\n",
        "        self.kdim = kdim if kdim is not None else embed_dim\n",
        "        self.vdim = vdim if vdim is not None else embed_dim\n",
        "\n",
        "        self.dropout = nn.Dropout(p_dropout)\n",
        "        self.softmax = nn.Softmax(dim=2)\n",
        "\n",
        "    def forward(self, query: Tensor, key: Tensor, value: Tensor, mask: Optional[Tensor] = None) -> Tuple[Tensor, Tensor]:\n",
        "          \n",
        "        \"\"\"\n",
        "        Calculate the attention weights.\n",
        "        q, k, v must have matching leading dimensions.\n",
        "        k, v must have matching penultimate dimension, i.e.: seq_len_k = seq_len_v.\n",
        "        The mask has different shapes depending on its type(padding or look ahead)\n",
        "        but it must be broadcastable for addition.\n",
        "\n",
        "        Args:\n",
        "            query : query shape == (..., seq_len_q, depth)\n",
        "            key : key shape == (..., seq_len_k, depth)\n",
        "            value: value shape == (..., seq_len_v, depth_v)\n",
        "            mask: Float tensor with shape broadcastable\n",
        "                to (..., seq_len_q, seq_len_k). Defaults to None.\n",
        "\n",
        "        Returns:\n",
        "            output, attention_weights\n",
        "        \"\"\"\n",
        "        qk = torch.matmul(query, torch.transpose(key,2,3))  # (..., seq_len_q, seq_len_k)\n",
        "        \n",
        "        # scale matmul_qk\n",
        "        scaled_attention_logits = qk / math.sqrt(self.kdim)\n",
        "\n",
        "        # add the mask to the scaled tensor.\n",
        "        if mask is not None:\n",
        "            scaled_attention_logits = scaled_attention_logits.masked_fill(mask == 1, -1e10)\n",
        "        \n",
        "        # softmax is normalized on the last axis (seq_len_k) so that the scores\n",
        "        # add up to 1.\n",
        "        attention_weights = self.softmax(scaled_attention_logits)  # (..., seq_len_q, seq_len_k)\n",
        "        \n",
        "        attention_weights = self.dropout(attention_weights)\n",
        "        \n",
        "        output = torch.matmul(attention_weights, value)  # (..., seq_len_q, depth_v)\n",
        "\n",
        "        return output"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-JAQLJrOVqGb"
      },
      "source": [
        "class _MultiHeadAttention(nn.Module):\n",
        "    \n",
        "    def __init__(self, embed_dim, num_heads, device, p_dropout=0.,\n",
        "                 kdim=None, vdim=None, epsilon = 1e-5):\n",
        "        super(_MultiHeadAttention, self).__init__()\n",
        "        \n",
        "        self.device = device \n",
        "        self.embed_dim = embed_dim                               \n",
        "        self.num_heads = num_heads\n",
        "        self.kdim = kdim * num_heads if kdim is not None else embed_dim\n",
        "        self.vdim = vdim * num_heads if vdim is not None else embed_dim\n",
        "        self.d_model = self.vdim \n",
        "        self.head_dim = self.d_model // num_heads\n",
        "        assert self.head_dim * num_heads == self.d_model, \"embed_dim must be divisible by num_heads\"\n",
        "\n",
        "        self.Wq = nn.Linear(embed_dim, self.kdim,device = device)\n",
        "        self.Wk = nn.Linear(embed_dim, self.kdim,device = device)\n",
        "        self.Wv = nn.Linear(embed_dim, self.vdim,device = device)\n",
        "\n",
        "        self.self_attention = _ScaledDotProductAttention(self.head_dim, device,p_dropout, kdim, vdim)\n",
        "        self.layernorm = nn.LayerNorm((embed_dim,), eps = epsilon, device = device)\n",
        "        self.linear = nn.Linear(self.vdim, embed_dim,device = device)\n",
        "        self.dropout = nn.Dropout(p_dropout)\n",
        "\n",
        "    def split_heads(self, x, batch_size):\n",
        "        \"\"\"\n",
        "        Split the last dimension into (num_heads, depth).\n",
        "        Transpose the result such that the shape is (batch_size, num_heads, seq_len, depth)\n",
        "        \"\"\"\n",
        "        x = x.view((batch_size, -1, self.num_heads, self.head_dim))\n",
        "        return torch.transpose(x,1,2)\n",
        "\n",
        "    def forward(self, query: Tensor, key: Tensor, value: Tensor, mask: Optional[Tensor] = None) -> Tuple[Tensor, Tensor]:\n",
        "        \n",
        "        batch_size = query.shape[0]\n",
        "\n",
        "        q = self.Wq(query)  # (batch_size, seq_len, d_model)\n",
        "        k = self.Wk(key)    # (batch_size, seq_len, d_model)\n",
        "        v = self.Wv(value)  # (batch_size, seq_len, d_model)\n",
        "\n",
        "        q = self.split_heads(q, batch_size)  # (batch_size, num_heads, seq_len_q, depth)\n",
        "        k = self.split_heads(k, batch_size)  # (batch_size, num_heads, seq_len_k, depth)\n",
        "        v = self.split_heads(v, batch_size)  # (batch_size, num_heads, seq_len_v, depth)\n",
        "\n",
        "        # scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)\n",
        "        # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)\n",
        "        scaled_attention = self.self_attention(q, k, v, mask)\n",
        "\n",
        "        scaled_attention = torch.transpose(scaled_attention, 1,2)  # (batch_size, seq_len_q, num_heads, depth)\n",
        "\n",
        "        concat_attention = scaled_attention.reshape((batch_size, -1, self.d_model))  # (batch_size, seq_len_q, d_model)\n",
        "\n",
        "        output = self.linear(concat_attention)  # (batch_size, seq_len_q, d_model)\n",
        "\n",
        "        output = self.dropout(output)\n",
        "        output = self.layernorm(value + output)\n",
        "        return output"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OGrjlrm9VqGb"
      },
      "source": [
        "class _PointwiseFeedicorward(nn.Module):\n",
        "    \n",
        "    def __init__(self, embed_dim, inner_dim,device, p_dropout=0.,epsilon = 1e-5):\n",
        "        super(_PointwiseFeedicorward, self).__init__()\n",
        "\n",
        "        self.linear1 = nn.Linear(embed_dim, inner_dim,device = device)\n",
        "        self.linear2 = nn.Linear(inner_dim, embed_dim,device = device)\n",
        "        self.layernorm = nn.LayerNorm((embed_dim,), eps = epsilon,device = device)\n",
        "        self.dropout = nn.Dropout(p_dropout)\n",
        "        self.relu = nn.ReLU()\n",
        "\n",
        "    def forward(self, x):\n",
        "        x1 = self.linear1(x)\n",
        "        x1 = self.relu(x1)\n",
        "        x1 = self.dropout(x1)\n",
        "        x1 = self.linear2(x1)\n",
        "        x1 = self.dropout(x1) \n",
        "        x1 = self.layernorm(x + x1)\n",
        "\n",
        "        return x1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S3EMmEIIVqGc"
      },
      "source": [
        "class _EncoderLayer(nn.Module):\n",
        "    \n",
        "    def __init__(self, embed_dim, inner_dim, num_heads, device, p_dropout = 0.,\n",
        "                 kdim = None, vdim = None,epsilon = 1e-5):\n",
        "        super(_EncoderLayer, self).__init__()\n",
        "\n",
        "        self.device = device\n",
        "        self.mha = _MultiHeadAttention(embed_dim, num_heads, device, p_dropout, kdim, vdim)\n",
        "        self.ff =  _PointwiseFeedicorward(embed_dim, inner_dim, device, p_dropout,epsilon = 1e-5)\n",
        "\n",
        "    def forward(self, x, mask = None):\n",
        "        \n",
        "        x = self.mha(x, x, x, mask)  # (batch_size, input_seq_len, d_model)\n",
        "        x = self.ff(x)         # (batch_size, input_seq_len, d_model)\n",
        "\n",
        "        return x"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pGkbtbarVqGc"
      },
      "source": [
        "class _TransformerEncoder(nn.Module):\n",
        "    \n",
        "    def __init__(self,num_layers, embed_dim,inner_dim, num_heads, device,\n",
        "                 p_dropout = 0., kdim = None, vdim = None, epsilon = 1e-5):\n",
        "        \n",
        "        super(_TransformerEncoder, self).__init__()\n",
        "\n",
        "        self.num_layers = num_layers \n",
        "        self.device = device\n",
        "        \n",
        "        kwrgs = {'p_droput': p_dropout, 'kdim': kdim, 'vdim': vdim, 'epsilon': epsilon}\n",
        "        # self.enc_layers = [_EncoderLayer(embed_dim, inner_dim, num_heads,device,**kwargs) for _ in range(num_layers)]\n",
        "        \n",
        "        self.enc_layers = nn.ModuleList([_EncoderLayer(embed_dim, inner_dim, num_heads,device,**kwargs) for _ in range(num_layers)])\n",
        "    def forward(self, x, mask):\n",
        "        \n",
        "        # for i in range(self.num_layers):\n",
        "        #     x = self.enc_layers[i](x, mask)\n",
        "        for layer in self.enc_layers:\n",
        "            x = layer(x,mask)\n",
        "        return x  # (batch_size, input_seq_len, d_model)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RBOeyeQVVqGd"
      },
      "source": [
        "class Transformer(nn.Module):\n",
        "\n",
        "    def __init__(self, num_layers, embed_dim, ff_dim, num_heads, src_vocab_size,\n",
        "                 trg_vocab_size, device, p_dropout = 0., kdim = None,\n",
        "                 vdim = None, maximum_position_encoding = 5000, epsilon = 1e-5):\n",
        "        \n",
        "        super(Transformer, self).__init__()\n",
        "\n",
        "        self.device = device\n",
        "\n",
        "        self.src_embedding = nn.Embedding(src_vocab_size, embed_dim)\n",
        "        self.pos_encoder = PositionalEncoding(embed_dim, p_dropout, maximum_position_encoding)\n",
        "        self.scale = embed_dim\n",
        "        kwrgs = {'p_droput': p_dropout, 'kdim': kdim, 'vdim': vdim, 'epsilon': epsilon}\n",
        "        self.transformer_encoder = _TransformerEncoder(num_layers, embed_dim, ff_dim,\n",
        "                                                       num_heads,device = device, **kwargs)\n",
        "        \n",
        "\n",
        "        self.trg_embedding = nn.Embedding(trg_vocab_size, embed_dim)\n",
        "        self.pos_decoder = PositionalEncoding(embed_dim, p_dropout, maximum_position_encoding)\n",
        "        self.decoder_layer = TransformerDecoderLayer(embed_dim, num_heads, dim_feedforward = ff_dim,\n",
        "                                                     dropout = p_dropout, layer_norm_eps = epsilon, \n",
        "                                                     batch_first = True, device = device)\n",
        "        self.transformer_decoder = TransformerDecoder(self.decoder_layer, num_layers)\n",
        "\n",
        "\n",
        "        self.trg_classifier = nn.Linear(embed_dim, trg_vocab_size)\n",
        "\n",
        "    def make_padding_mask(self, seq, pad_idx):\n",
        "        seq_mask = seq == pad_idx\n",
        "        return seq_mask.to(self.device)\n",
        "    \n",
        "    def make_src_padding_mask(self, seq, pad_idx):\n",
        "        seq_mask = seq == pad_idx\n",
        "        return seq_mask.view(seq.shape[0],1,1,-1).to(self.device)\n",
        "\n",
        "    def generate_square_subsequent_mask(self, sz: int) -> Tensor:\n",
        "        r\"\"\"Generate a square mask for the sequence. The masked positions are filled with float('-inf').\n",
        "            Unmasked positions are filled with float(0.0).\n",
        "        \"\"\"\n",
        "        mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)\n",
        "        mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))\n",
        "        return mask.to(self.device)\n",
        "\n",
        "    def forward(self, src, trg):\n",
        "        \n",
        "        src_padding_mask = self.make_src_padding_mask(src, 0)\n",
        "        mem_padding_mask = self.make_padding_mask(src, 0)\n",
        "        src = self.src_embedding(src).to(self.device) * math.sqrt(self.scale)\n",
        "        src = self.pos_encoder(src).to(self.device)\n",
        "\n",
        "        enc_output = self.transformer_encoder(src, src_padding_mask) # (batch_size, src_seq_len, d_model)\n",
        "\n",
        "\n",
        "        trg_padding_mask = self.make_padding_mask(trg, 0)\n",
        "        trg_mask = self.generate_square_subsequent_mask(trg.shape[1])\n",
        "        kwargs = {'tgt_mask': trg_mask, 'tgt_key_padding_mask': trg_padding_mask, 'memory_key_padding_mask': mem_padding_mask}\n",
        "        trg = self.trg_embedding(trg).to(self.device) * math.sqrt(self.scale)\n",
        "        trg = self.pos_decoder(trg).to(self.device)\n",
        "\n",
        "        dec_output = self.transformer_decoder(trg, enc_output,**kwargs) # (batch_size, trg_seq_len, d_model)\n",
        "\n",
        "\n",
        "        final_output = self.trg_classifier(dec_output) # (batch_size, trg_seq_len, trg_vocab_size)\n",
        "\n",
        "        return final_output \n",
        "\n",
        "    # function to generate output sequence using greedy algorithm\n",
        "    def greedy_decode(self, src, max_len, sos_idx, eos_idx):\n",
        "        src = src.to(self.device)\n",
        "\n",
        "        # memory = self.encode(src).to(device)\n",
        "        y_pred = torch.ones(1, 1).fill_(sos_idx).type(torch.long).to(self.device)\n",
        "\n",
        "        for i in range(max_len-1):\n",
        "            \n",
        "            trg_mask = (self.generate_square_subsequent_mask(y_pred.size(1)).type(torch.bool)).to(self.device)\n",
        "            out = self.decode(src, y_pred, trg_mask)\n",
        "            # out = out.transpose(0, 1)\n",
        "            prob = self.trg_classifier(out[0, :, -1])\n",
        "            _, next_word = torch.max(prob, dim=0)\n",
        "            next_word = next_word.item()\n",
        "            y_pred = torch.cat([y_pred, torch.ones(1, 1).type_as(src.data).fill_(next_word)], dim=1)\n",
        "            if next_word == eos_idx:\n",
        "                break\n",
        "        return y_pred\n",
        "\n",
        "    def encode(self, src, mask):\n",
        "        src = self.src_embedding(src).to(self.device) * math.sqrt(self.scale)\n",
        "        src = self.pos_encoder(src)\n",
        "\n",
        "        enc_output = self.transformer_encoder(src, mask) # (batch_size, src_seq_len, d_model)\n",
        "        return enc_output\n",
        "    \n",
        "    def decode(self, trg, src_enc, trg_mask, mem_padding_mask):\n",
        "        kwargs = {'tgt_mask': trg_mask,'memory_key_padding_mask': mem_padding_mask}\n",
        "        \n",
        "        trg = self.trg_embedding(trg).to(self.device) * math.sqrt(self.scale)\n",
        "        trg = self.pos_decoder(trg)\n",
        "        # dec_output.shape == (batch_size, trg_seq_len, d_model)\n",
        "        dec_output = self.transformer_decoder(trg, src_enc, **kwargs)\n",
        "        final_output = self.trg_classifier(dec_output) # (batch_size, trg_seq_len, trg_vocab_size)\n",
        "\n",
        "        return final_output\n",
        "\n",
        "    def translate_sentence(self,src,max_len, sos_idx, eos_idx):\n",
        "        src = src.to(self.device)\n",
        "        src_padding_mask = self.make_src_padding_mask(src, 0)\n",
        "        mem_padding_mask = self.make_padding_mask(src, 0)\n",
        "\n",
        "        enc_src = self.encode(src, src_padding_mask)\n",
        "\n",
        "        trg_indexes = [sos_idx]\n",
        "\n",
        "        for i in range(max_len):\n",
        "\n",
        "            trg = torch.LongTensor(trg_indexes).unsqueeze(0).to(device)\n",
        "            trg_mask = self.generate_square_subsequent_mask(trg.shape[1])\n",
        "\n",
        "            output = self.decode(trg, enc_src, trg_mask, mem_padding_mask)\n",
        "            \n",
        "            pred_token = output.argmax(2)[:,-1].item()\n",
        "            \n",
        "            trg_indexes.append(pred_token)\n",
        "\n",
        "            if pred_token == eos_idx:\n",
        "                break\n",
        "        \n",
        "        return trg_indexes"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-Mad73SBKQ5O"
      },
      "source": [
        "## Sanity Check \n",
        "(try to overfit on a mini batch)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "referenced_widgets": [
            "d5f79143553d45f2aeff4eee1b4cc520"
          ]
        },
        "id": "46JKXdXfKQ5c",
        "outputId": "5b1d5c2a-25e8-4e1c-ab9b-65e577be247d"
      },
      "source": [
        "torch.manual_seed(2045)\n",
        "\n",
        "pad_idx = 0\n",
        "\n",
        "# Parameters\n",
        "params = {'batch_size': 128,\n",
        "          'shuffle': True,\n",
        "          'num_workers':2,\n",
        "          'collate_fn' : BatchPadCollate(pad_idx=pad_idx)\n",
        "          }\n",
        "\n",
        "# Training and validation Data Generators\n",
        "training_set2 = AFEC_Dataset(en_train_dic_,fa_train_dic_,partition2['train'],en_vocab_dic_, fa_vocab_dic_)\n",
        "training_generator2 = torch.utils.data.DataLoader(training_set2, **params)\n",
        "\n",
        "validation_set2 =  AFEC_Dataset(en_train_dic_,fa_train_dic_,partition2['validation'],en_vocab_dic_, fa_vocab_dic_)\n",
        "validation_generator2 = torch.utils.data.DataLoader(validation_set2, **params)\n",
        "\n",
        "# Hyperparameters\n",
        "embed_dim = 256\n",
        "num_heads = 8\n",
        "dim_key = 64\n",
        "dim_value = 64\n",
        "dim_inner = 1024\n",
        "num_layers = 3\n",
        "src_vocab_size = len(en_vocab_dic_.get_str_2_idx())\n",
        "trg_vocab_size = len(fa_vocab_dic_.get_str_2_idx())\n",
        "learning_rate = 1e-4\n",
        "num_iterations = 400\n",
        "\n",
        "kwargs = {'p_dropout': 0.1,\n",
        "          'kdim': dim_key,\n",
        "          'vdim':dim_value,\n",
        "          }\n",
        "# Model\n",
        "model2 = Transformer(num_layers, embed_dim, dim_inner, num_heads, src_vocab_size,trg_vocab_size, device,**kwargs).to(device)\n",
        "\n",
        "# Define Loss function and Optimizer\n",
        "criterion = nn.CrossEntropyLoss(ignore_index = pad_idx)\n",
        "optimizer = optim.Adam(model2.parameters(), lr = learning_rate, betas=(0.9, 0.98), eps=1e-9)\n",
        "\n",
        "train_losses2 = []\n",
        "validation_losses2 = []\n",
        "\n",
        "torch.autograd.set_detect_anomaly(True)\n",
        "\n",
        "train_dataiter2 = iter(training_generator2)\n",
        "batch_train_src, batch_train_trg = train_dataiter2.next()\n",
        "dataiter2 = iter(validation_generator2)\n",
        "batch_src, batch_trg = dataiter2.next() \n",
        "\n",
        "trainloss, valloss, running_trainloss, running_valloss = 0.0, 0.0, 0.0, 0.0\n",
        "\n",
        "# Loop over epochs\n",
        "for itr in tqdm(range(num_iterations)):\n",
        "\n",
        "    # Training        \n",
        "    # Transfer to GPU\n",
        "    batch_train_src, batch_train_trg = batch_train_src.to(device), batch_train_trg.to(device)\n",
        "\n",
        "    # zero the parameter gradients\n",
        "    optimizer.zero_grad()\n",
        "    \n",
        "    # Model computations\n",
        "    # forward + backward + optimize\n",
        "    batch_train_outputs = model2(batch_train_src, batch_train_trg[: , :-1])\n",
        "    \n",
        "    loss = criterion(torch.transpose(batch_train_outputs,1,2), batch_train_trg[: , 1:])\n",
        "\n",
        "    loss.backward()\n",
        "\n",
        "    # Clip to avoid exploding gradient issues, makes sure grads are within a healthy range\n",
        "    nn.utils.clip_grad_norm_(model2.parameters(), max_norm=2,error_if_nonfinite= False)\n",
        "\n",
        "    optimizer.step()\n",
        "\n",
        "    # print statistics\n",
        "    trainloss = loss.item()\n",
        "    running_trainloss += loss.item()\n",
        "\n",
        "    if itr % 100 == 99:    # print every 100 mini-batches\n",
        "        print(\"Cost of Train data after %i iterations : %f\" %(itr + 1, running_trainloss / 100))\n",
        "        running_trainloss = 0\n",
        "    \n",
        "    # Validation\n",
        "    with torch.set_grad_enabled(False):\n",
        "        # Transfer to GPU\n",
        "        batch_src, batch_trg = batch_src.to(device), batch_trg.to(device)\n",
        "        \n",
        "        # Model computations\n",
        "        # forward + backward + optimize\n",
        "        batch_outputs = model2(batch_src, batch_trg[: , :-1])\n",
        "        \n",
        "        val_loss = criterion(torch.transpose(batch_outputs,1,2), batch_trg[: , 1:])\n",
        "\n",
        "        valloss = val_loss.item()\n",
        "        running_valloss += val_loss.item()\n",
        "\n",
        "        if itr % 100 == 99:    # print every 100 mini-batches\n",
        "            print(\"Cost of Validation data after %i iterations: %f\" %(itr + 1, running_valloss / 100))\n",
        "            running_valloss = 0\n",
        "\n",
        "    train_losses2.append(trainloss)\n",
        "    validation_losses2.append(valloss)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d5f79143553d45f2aeff4eee1b4cc520",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=400.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Cost of Train data after 100 iterations : 4.961051\n",
            "Cost of Validation data after 100 iterations: 5.268263\n",
            "Cost of Train data after 200 iterations : 1.881628\n",
            "Cost of Validation data after 200 iterations: 3.707742\n",
            "Cost of Train data after 300 iterations : 0.837785\n",
            "Cost of Validation data after 300 iterations: 4.104653\n",
            "Cost of Train data after 400 iterations : 0.340454\n",
            "Cost of Validation data after 400 iterations: 4.525049\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "HT9hTTi5KQ5e",
        "outputId": "dbbd441c-455c-45ce-8a7e-121fb1f23dbf"
      },
      "source": [
        "# plot the cost\n",
        "plt.plot(train_losses2)\n",
        "plt.plot(validation_losses2)\n",
        "plt.title(\"Cost of Transfromer vs iters \")\n",
        "plt.ylabel('Cost')\n",
        "plt.xlabel('Iterations')\n",
        "plt.legend(labels = [\"Train\",\"Validation\"])\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEWCAYAAABhffzLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3xW1f3A8c83T/aEDAgQIOw9DSBDxbpQKTgQxQVitWq1Vmut9qfVOmpbtVWrdStWUVRUxL0VKsiUvTeBQBYZZI/z++PcQAhJSCB5bpLn+369nlee587vc5J877nnnnuuGGNQSinlO/zcDkAppZR3aeJXSikfo4lfKaV8jCZ+pZTyMZr4lVLKx2jiV0opH6OJXzUJIjJaRDaLyEERucDteKoSkbYiMk9EckXkcbfj8QYR6eT8Pjxux6IaliZ+HyQil4vIUuefOkVEPhORMSe4zR0icuYJbOIB4GljTLgxZk6VbR+s9CoXkYJKn684kbjr4XogHYg0xvzeS/t0lTFml/P7KAMQke9F5Fdux6VOnCZ+HyMitwNPAH8F2gKdgP8AE92MC+gMrK1uhpN8wo0x4cAu4JeVps2sWE5E/Bs5vnWmhjseG3nftXJz3/WhZw5NiDFGXz7yAqKAg8AltSwThD0w7HVeTwBBzrxY4GMgC8gE5mMrD68D5UCBs/07a9j2dcAWZ925QHtn+tYq6wfVEt8O4Ezn/VggGfgjsM+Jo7UTYxpwwHmfUGn974EHgR+BXOBLINaZFwy8AWQ433EJ9uA4AygBip34zgTuB2Y7y+cAvwLaO98r0/me11Xa7/3Au87yucBqoCdwN5AK7AbOrvK7ehlIAfYADwEeZ940J/5/ObE+VKWM2jtlGV1p2hDsGUsA0B34Ach2pr1dQ1knAgbwBx4GyoBCpwyedpbpDXzlfOeNwORK688AngU+BfKccjsPWOeUwR7gDrf/L3zx5XoA+vLiLxvGAaWAfy3LPAD8BLQB4oAFwIPOvEeA55zkEQCcAogzbwdOQq5hu79wksxQ7MHl38C8SvNrXb+65bCJvxT4u7PNECAGuBgIBSKcZDun0vrfYw80PZ3lvwf+5sz7NfCRs64HOAnbtFORxB6qtJ37sQeDC7AHvxBgHvbsKRgYjD34/KLS8oXAOU4i/S+wHfg/pyyvA7ZX2v4HwPNAmPO7WAz82pk3zfnetzjbCqmmnL7lyAPPo8Bzzvu3nP36ObGOqaGsE3ESf6Wy+1Wl+WHYA9Y1ThwVB5e+lcosGxhdaV8pwCnO/NbAULf/L3zxpU09viUGSDfGlNayzBXAA8aYVGNMGvAX4CpnXgnQDuhsjCkxxsw3zn9wHVwBvGKMWW6MKcLWdEeKSOLxfJFKyoH7jDFFxpgCY0yGMeY9Y0y+MSYXW1M9rco6rxpjNhljCoB3sEm64vvFAN2NMWXGmGXGmJxa9r3QGDPHGFOOPRsaDfzRGFNojFkBvARcXWn5+caYL5zyfxd7YP2bMaYEmAUkikgrEWmLrRn/zhiTZ4xJxdbuL6u0rb3GmH8bY0qd71HVm8AUABERZ903K33PztgzrkJjzP9q+Y61GQ/sMMa86sTxM/AecEmlZT40xvxojCk3xhQ6++4rIpHGmAPGmOXHuW91AjTx+5YMIPYYbcLtgZ2VPu90poGtNW4BvhSRbSJyVz32fcR2jTEHnXg61GMb1UlzEgoAIhIqIs+LyE4RycHWwltVaV/eV+l9PhDuvH8d+AKYJSJ7ReQfIhJQy753V3rfHsh0DjYVdnLk99tf6X0B9iBcVukzTiydsWcBKSKSJSJZ2Np/mxr2XZ33sAfWdsCp2APkfGfenYAAi0VkrYhMP8a2atIZGFERoxPnFUB8LXFejD2o7RSRH0Rk5HHuW50ATfy+ZSFQhG2eqMle7D90hU7ONIwxucaY3xtjugITgNtF5AxnuWPV/I/YroiEYWvXe+r1DY5Wdb+/B3oBI4wxkdikBzbR1b4hexbzF2NMX2AUtkZ7dW2rVHq/F4gWkYhK0zpxfN9vN/b3FGuMaeW8Io0x/WrY99GBGXMAe/3iUuByYFbF2ZkxZp8x5jpjTHts89Z/RKR7HeKqus/dwA+VYmxl7AX3G2taxxizxBgzEXsQm4M941JeponfhxhjsoE/A8+IyAVO7ThARM4VkX84i70F3CMicSIS6yz/BoCIjBeR7k7TQTb2Yl+5s95+oGstu38LuEZEBotIELZX0SJjzI4G/poR2NpzlohEA/fVdUUROV1EBjhnBznYZonyY6wGgDFmN/Z6yCMiEiwiA4FrccquPowxKdik/biIRIqIn4h0E5GqTVbH8ib2wDWJw808iMglIpLgfDyATc51+Z5Vf8cfAz1F5Crn7yhARIaJSJ/qVhaRQBG5QkSinOatnDruVzUwTfw+xhjzOHA7cA/24uNu4GZs7Qts75GlwCpsz5PlzjSAHsDX2F4dC4H/GGO+c+Y9gj1gZInIHdXs92vgXmwTRArQjSPbrBvKE9gLrenYi9Sf12PdeGxPnRxgPbbny+v1WH8K9oLoXuzF2fuc7308rgYCsT1gDjhxtavnNuZif2f7jDErK00fBiwSkYPOMrcaY7bVYXtPApNE5ICIPOU0a52N/T3uxTahVVxor8lVwA6nGe4GbNOQ8rKKHhlKKaV8hNb4lVLKx2jiV0opH6OJXymlfIwmfqWU8jHNYnCn2NhYk5iY6HYYSinVrCxbtizdGBNXdXqzSPyJiYksXbrU7TCUUqpZEZGd1U3Xph6llPIxmviVUsrHaOJXSikf0yza+JVSLUdJSQnJyckUFhYee2FVJ8HBwSQkJBAQUNtgsodp4ldKeVVycjIREREkJiZix/tTJ8IYQ0ZGBsnJyXTp0qVO62hTj1LKqwoLC4mJidGk30BEhJiYmHqdQWniV0p5nSb9hlXf8mzRiX/Oz3t446dqu7EqpZTPatGJ/5PVKZr4lVJHyMjIYPDgwQwePJj4+Hg6dOhw6HNxcXGt6y5dupTf/va3Xoq08bToi7ux4UH8vOuA22EopZqQmJgYVqxYAcD9999PeHg4d9xx+NlBpaWl+PtXnxqTkpJISkrySpyNqUXX+GPDA8nMK6asXB82o5Sq2bRp07jhhhsYMWIEd955J4sXL2bkyJEMGTKEUaNGsXHjRgC+//57xo8fD9iDxvTp0xk7dixdu3blqaeecvMr1EuLr/GXG8jMKyYuoranwSml3PCXj9aybm9Og26zb/tI7vtlv2MvWEVycjILFizA4/GQk5PD/Pnz8ff35+uvv+ZPf/oT77333lHrbNiwge+++47c3Fx69erFjTfeWOe+9G5q8YkfICOvSBO/UqpWl1xyCR6PB4Ds7GymTp3K5s2bERFKSkqqXef8888nKCiIoKAg2rRpw/79+0lISKh22aak0RK/iLwCjAdSjTH9nWnRwNvYB1LvACYbYxqtET42PBCA9Nxi+xhtpVSTcjw188YSFhZ26P29997L6aefzgcffMCOHTsYO3ZstesEBR2uUHo8HkpLSxs7zAbRmG38M4BxVabdBXxjjOkBfON8bjSxTi0//WBRY+5GKdXCZGdn06FDBwBmzJjhbjCNoNESvzFmHpBZZfJE4DXn/WvABY21fzjc1KOJXylVH3feeSd33303Q4YMaTa1+PoQYxqvx4uIJAIfV2rqyTLGtHLeC3Cg4nNtkpKSzPE8iMXMns7CVRv5YeRL3H1un3qvr5RqeOvXr6dPH/1/bGjVlauILDPGHNX/1LXunMYecWo86ojI9SKyVESWpqWlHdc+BKGjJ8O28SullAK8n/j3i0g7AOdnak0LGmNeMMYkGWOS4uKOemRk3YS3JYYsbepRSqlKvJ345wJTnfdTgQ8bdW/hbQg1BRzMzWrU3SilVHPSaIlfRN4CFgK9RCRZRK4F/gacJSKbgTOdz40nvC0A5mCNJxZKKeVzGq0fvzFmSg2zzmisfR4lvA0A/vlplJcb/Px0KFillGrRY/VU1PijTRbZBdXfeaeUUr7GJxJ/nOgFXqWUdfrpp/PFF18cMe2JJ57gxhtvrHb5sWPHUtGd/LzzziMr6+hrhvfffz+PPfZYrfudM2cO69atO/T5z3/+M19//XV9w28QLTvxh0ZjxEOcZJGmiV8pBUyZMoVZs2YdMW3WrFlMmVJT6/Rhn376Ka1aHfPWo2pVTfwPPPAAZ5555nFt60S17MTv56EsJJY4skk/qH35lVIwadIkPvnkk0MPXdmxYwd79+7lrbfeIikpiX79+nHfffdVu25iYiLp6ekAPPzww/Ts2ZMxY8YcGrYZ4MUXX2TYsGEMGjSIiy++mPz8fBYsWMDcuXP5wx/+wODBg9m6dSvTpk1j9uzZAHzzzTcMGTKEAQMGMH36dIqKig7t77777mPo0KEMGDCADRs2NEgZtOjROQEIb0Ncbha7crXGr1ST89ldsG91w24zfgCcW3OHwejoaIYPH85nn33GxIkTmTVrFpMnT+ZPf/oT0dHRlJWVccYZZ7Bq1SoGDhxY7TaWLVvGrFmzWLFiBaWlpQwdOpSTTjoJgIsuuojrrrsOgHvuuYeXX36ZW265hQkTJjB+/HgmTZp0xLYKCwuZNm0a33zzDT179uTqq6/m2Wef5Xe/+x0AsbGxLF++nP/85z889thjvPTSSydcRC27xg94IuNpI9naxq+UOqRyc09FM88777zD0KFDGTJkCGvXrj2iWaaq+fPnc+GFFxIaGkpkZCQTJkw4NG/NmjWccsopDBgwgJkzZ7J27dpaY9m4cSNdunShZ8+eAEydOpV58+Ydmn/RRRcBcNJJJ7Fjx47j/cpHaPE1fglvS1u/5Zr4lWqKaqmZN6aJEydy2223sXz5cvLz84mOjuaxxx5jyZIltG7dmmnTplFYWHhc2542bRpz5sxh0KBBzJgxg++///6EYq0Y+rkhh31u8TV+wtsQTTYZucf3S1RKtTzh4eGcfvrpTJ8+nSlTppCTk0NYWBhRUVHs37+fzz77rNb1Tz31VObMmUNBQQG5ubl89NFHh+bl5ubSrl07SkpKmDlz5qHpERER5ObmHrWtXr16sWPHDrZs2QLA66+/zmmnndZA37R6PpH4AyilMDfD7UiUUk3IlClTWLlyJVOmTGHQoEEMGTKE3r17c/nllzN69Oha1x06dCiXXnopgwYN4txzz2XYsGGH5j344IOMGDGC0aNH07t370PTL7vsMh599FGGDBnC1q1bD00PDg7m1Vdf5ZJLLmHAgAH4+flxww03NPwXrqRRh2VuKMc7LDMAa96D2dO5IuAJZv7fNQ0bmFKq3nRY5sbRLIZl9hrnJi7/gjSaw0FOKaUaW8tP/BHtAIgpzyCnoOU9SUcpperLZxJ/vBxgv17gVapJ0LPvhlXf8mz5iT8wlNKgVrSTDPZmFbgdjVI+Lzg4mIyMDE3+DcQYQ0ZGBsHBwXVep8X34wcoj2hPfH4m+7K1xq+U2xISEkhOTuZ4H6mqjhYcHExCQkKdl/eJxO/fOoF2qZtYo4lfKdcFBATQpUsXt8PwaS2/qQfwi+xAe79MUrK1qUcppXwi8RPZgWhySM/KcTsSpZRynY8k/vYAlGbtcTkQpZRyn28k/qgOAPjlprgciFJKuc83En+kTfxRJankFuqzd5VSvs03Er9zE1d70S6dSinlG4k/KJzSwEjiJYO9mviVUj7ONxI/UB7RgXaSyT7t0qmU8nE+k/j9W3egnWSQojV+pZSP85nE79eqE5380nW8HqWUz/OZxE/rLkRxkAOZOj6IUsq3+U7ij3bGBsnc7m4cSinlMt9J/K1t4g85uIuych0OVinlu3wo8ScCkGD2k6oPZFFK+TBXEr+I3CYia0VkjYi8JSJ1f4LA8QoKpzg4lk6yn+QDeoFXKeW7vJ74RaQD8FsgyRjTH/AAl3lj32WtEuksqezOzPfG7pRSqklyq6nHHwgREX8gFNjrjZ0GxnWjs99+tqfneWN3SinVJHk98Rtj9gCPAbuAFCDbGPNl1eVE5HoRWSoiSxvqEW2e6C7ESyY792c2yPaUUqo5cqOppzUwEegCtAfCROTKqssZY14wxiQZY5Li4uIaZufRXfDDkJ+qXTqVUr7LjaaeM4Htxpg0Y0wJ8D4wyit7drp0StYO7dKplPJZbiT+XcDJIhIqIgKcAaz3yp5jugPQqTxZh25QSvksN9r4FwGzgeXAaieGF7yy87AYikPi6C272Zp20Cu7VEqppsaVXj3GmPuMMb2NMf2NMVcZY4q8tvM2fejpl8y2NO3Zo5TyTb5z564joF1/evolsz0t2+1QlFLKFT6X+KVNX0IoJjdlq9uhKKWUK3wu8dO2LwD+6RtcDkQppdzhe4k/rjcGoX3RNvbn6GBtSinf43uJPzCMoohO9PTbzapkbedXSvke30v8QEC7fvTx283q5Cy3Q1FKKa/zycTvSTiJbrKX7bt2uh2KUkp5nU8mfrqcCkDI3p8wRoduUEr5Ft9M/O2HUOIJoV/xKvZm6wVepZRv8c3E7wmgIH44I/3WsWq3tvMrpXyLbyZ+ILTXWHr67WH9Fr2RSynlW3w28ft3te38RVvmuRyJUkp5l88mftoNpsgTRsecpWQXlLgdjVJKeY3vJn6PP/nxwxgp61iyXR/FqJTyHb6b+IHwXmPp5pfCmg06bo9Synf4dOIP6HW2/bn1c5cjUUop7/HpxE+bvmSGdGZI7vek5XrvWTBKKeUm3078IpT1nsgIWc+Cld557K9SSrnNtxM/EDviUjxiyF7+vtuhKKWUV/h84pe2/cgITmRA+ifsyypwOxyllGp0Pp/4EcGcfAND/Law6Fut9SulWj5N/EDsmOlk+bUmdu2rlJfraJ1KqZZNEz+AfxCp3S/h5NKlLP55udvRKKVUo9LE7+g87hZKJICyr//idihKKdWoNPE7gqI7sa7LNEYX/MCmJV+5HY5SSjUaTfyV9LzoHvYTTeBXf4LycrfDUUqpRqGJv5LwiChW9b6NxOJNbP3iGbfDUUqpRqGJv4pTLrqJpX4DSVj0AKXJP7sdjlJKNThN/FUEB/qTdd5zpJsICt+4HPIy3A5JKaUalCb+apxxUl/+E3cfAYVpFM6aBmWlboeklFINxpXELyKtRGS2iGwQkfUiMtKNOGoiIvzqskk8xLUE755H2Vf3uR2SUko1GLdq/E8CnxtjegODgCY3NGaX2DDGXHIb/y09C89PT8OKN90OSSmlGoTXE7+IRAGnAi8DGGOKjTFZ3o6jLs7pF8/K/nfxY3l/yufeCtv1wexKqebPjRp/FyANeFVEfhaRl0QkrOpCInK9iCwVkaVpaWnej9Lx5wmDeDjsLraXt6H8rSmQssq1WJRSqiG4kfj9gaHAs8aYIUAecFfVhYwxLxhjkowxSXFxcd6O8ZCo0ACeumYs15bdxYGyYMzMSXBgh2vxKKXUiXIj8ScDycaYRc7n2dgDQZPVvU0EN00Yy6X5d1JYkA+vXwgH3TsLUUqpE+H1xG+M2QfsFpFezqQzgHXejqO+Jg/ryJiRY7gi//eUZu2FNy6E7D1uh6WUUvXmVq+eW4CZIrIKGAz81aU46uWe8/sQ0m0kvy7+HWXp22DG+VrzV0o1O3VK/CLyel2m1ZUxZoXTfj/QGHOBMebA8W7Lm/w9fjxz+VC2RJ3MteYeTO4+mDkJig66HZpSStVZXWv8/Sp/EBEPcFLDh9P0tQoN5MWrk1hS0pUHQ+/E7FsN71wFpcVuh6aUUnVSa+IXkbtFJBcYKCI5zisXSAU+9EqETVDPthH869LBvJLai1nxv4et38Lcm3UoZ6VUs1Br4jfGPGKMiQAeNcZEOq8IY0yMMeZuL8XYJJ3dL57bz+rJ3dsHs7TrTbDqbfjmfrfDUkqpY6prU8/HFTdZiciVIvJPEenciHE1Czef3p1z+8czef1o9va4HH58En561u2wlFKqVnVN/M8C+SIyCPg9sBX4b6NF1Uz4+QmPXTKInm0jOW/zBPK6ngef3w1r3nM7NKWUqlFdE3+pMcYAE4GnjTHPABGNF1bzERbkz4tXJ4Gfh0lp0ylNGAFzboK9+hAXpVTTVNfEnysidwNXAZ+IiB8Q0HhhNS8do0P5z+VD2ZRRwh1+d2DCYmHWldrHXynVJNU18V8KFAHTnTtvE4BHGy2qZmhU91juPb8PczYVMzPxEchPh3enQlmJ26EppdQR6pT4nWQ/E4gSkfFAoTHG59v4q5o6KpGLhybw58Ueto96BHb+CF/8ye2wlFLqCHW9c3cysBi4BJgMLBKRSY0ZWHMkItw/oS/tokKYvrwrJcNvgsUvwM9vuB2aUkodUtemnv8DhhljphpjrgaGA/c2XljNV0RwAI9OGsj29DweKb0MupwGH98GyUvdDk0ppYC6J34/Y0xqpc8Z9VjX54zqHsvVIzvz6sJklg17HCLi4e0rIX2L26EppVSdk/fnIvKFiEwTkWnAJ8CnjRdW8/fHcb3p2DqU2z5OpmDSG/Yi74zzIXef26EppXzcscbq6S4io40xfwCeBwY6r4XAC16Ir9kKC/Ln0UkD2ZWZz9+X+8PUuVCUA29OhsJst8NTSvmwY9X4nwByAIwx7xtjbjfG3A584MxTtRjRNYZpoxKZsWAHi/PbwSWvwf618OalUJzvdnhKKR91rMTf1hizuupEZ1pio0TUwtw5rhcdo0O4c/ZKChLPgItehF0/wXvXQnGe2+EppXzQsRJ/q1rmhTRkIC1VaKA/f794IDsy8nn8y43Q/yI471HY+Bm8eAakbXI7RKWU2759CN6/3mu7O1biXyoi11WdKCK/ApY1Tkgtz6husVwxohMv/7idZTszYfh1cOV7kJcKL54OO/7ndohKqcaWvhmWzQBjjpxenA/zHrVDu6dtPHJeI935L6ZqEJVnirTFtucXczjRJwGBwIXOHb2NLikpySxd2rz7wR8sKuWcf81DBD66eQytwwLtw9pfvxCydsIvn4RBl7kdplKqISx9FbJ3Q+IpsPw16H6WvZkzZQUMmAxn3g9RHew1v5+ehZ+dJ9kOvhK6nQ4FB6Btf/jg1zBlFrTte1xhiMgyY0zSUdNrS/yVVj4d6O98XGuM+fa4ojhOLSHxA6zYncXk5xYyoms0M64ZjsdP7EBus6+BHfNhwr9h6NVuh6mU7ygrBT8PiNR/3XUfwup3YcQNkDgGclIgpDUU5cJj3Wtezz8YSgshqqM9OAAMudKuu+DfRy7bqhPcuBCCwusfHzUn/rqO1fOdMebfzsurSb8lGdyxFX+Z2I/5m9P551fOKV14nG326XYGfHw7rJ7tbpBKtVRFuXBgx+HPhTnweC/43z+PXjZtI3x4s73psqQQUlbCc6ccboopL4NP/wDrP4IPfwNbv4MnBsBf2x9O+j3H2Z/j/wUR7cATCL9ZAtd8Ch2SDif9m5fCxGfg9Hvg1D/AOY9AaCwEhsMFzx130q9NnWr8bmspNf4Kd723illLdvP8VSdxTr94O7EgC966DHYthOHXw9kPg3+gu4Eq1VwVHYSFT0PPc6D9EDvtk9/DkpfgF/fA6Nvgszth6ct2Xu/xED/QXn/b9h18dpe9BgcQHAWBEZCTbGvgI260d+PPvga6nArb5x2579AYiO0FUz+C/Wug/WB7oCgrgYDgw8tt+hJMOfQad3T8een2QBEceULFcEJNPW5raYm/sKSMS59fyNa0PN667mQGJETZGWUl8PX99g+20yi49A0Ii3E1VqUaVXk5+NWh4aGkEA7ug9aJh6eVlYD42aaaynYvgS/vgd0/gXjg5iUQEAKvngcHtttlortB5lZbqw6KhNy9R++zxzmw+QvwC4DyEpvQ8zMOz/cEwR0bYcMnNrG37gydRoJ/UL2LobFo4m9i9mYVcMEzP5KZV8zr145gZLdKCX71bPsUr8j2cMmrh2ssSrUk2XvgpTNhxPUw5jbb20XEJnSP85ynshLbnPL+9Tb5nvZH6HQyxA+CGefZtvJBU2DbD5CdDG37weYvIaQVjLoFvnkQTNnhfZ56J2RsgbXv27b5U/8AYbG2rf/NS2BrpZbse9KgvNQeLJ4dBROehsh29v9z1dv285ArvFtm9aSJvwk6kFfMxc8uIL+4jM9/dwqtQis17exeDO9Og4OpMOLXcOod9uKPUs2NMbD5K5uwK5ouNn5un1WRudV+HjTFLtM6EfYste3jJ99kH2ZUcACCoqCo0lAnkQn2YUeR7SFz25H7i+4K1/9g9/XRrbDqHShx7pS//F3ba2bvz5Aw7MiLuqXFsOkzG0NpEXQcfnheYbY9M6hYvjDbNgE1cZr4m6jVydlc9OyPjOkeywtXJxHgqXTam59pH96++l1o0wemvGXbGJXypsJsCAg9XAuvrLwcfnoGDuyEc/5qa9fpmyEqwSbkikEJ377CJtSbfrJ/z3NvsZ/H3AZbvra1+goVvV78gyEkGs5/HLqfCQ/FHV7GEwSTXoEeZ9n7YOJ6266R719nm0i7nHI4PlMO6Rvh24fhohca5WJpU6WJvwl746ed3DNnDb8c1J4nLh1su3lWtvVbeGeq/ceb8jZ0HOZOoKrlKy+zteOgCNsLJmWlPfPsfT6c+6hN7AEh8OOTtobe55f2IinAsOugMMsm9pp0PBn2LLPdHy9/+3B7eFmpHcTwxydh5M2w4Cn7OvMvMOZ3dpmv7oN1c+CW5VBWbONQtdLE38Q998NW/vbZBiYnJfC3iwbiVzX5p2+xN3tl77Y3eg27DjoMPb7+x0qBvQgaFgsFmbZykZNi28eLD9oeKTMnH77oKR57rWnPUghuZRO8nWFr7l1OtTcqVRhyFXT7hW2m2fiZ/Ztd/S5s+txeWL3um9qbLsvLbExdx1Z/pqHqRBN/M/DPrzbx1DebmTqyM/dP6IdUTeoFWTD/cVj0PJQV2b7/k1+ztTOlKmz5Gvz8bdI0xl70/PkNeyFyz3Lb+2T7fPiqykP0AsMhvO3hdnewTSxRHWHZq4endTnNNp90P8P2QjvtLlvzf2607YJ45/bqm1NKi2DnAtu9Ua9XeUWTS/wi4gGWAnuMMeNrW9ZXEr8xhr9+up4X52/nhtO68cdxvY5O/mAPAMtfs6e+ARFJlxkAABtiSURBVCHQZ4I9HY7rrWcAvm7dh/COc/d30rWwe5HtS16bNn3h6rk2GXv84ZM7YMmLtmJx1ft2mcUvwjcP2Db6qA6H101Zadf3BNgLtu0G2Z4vqkloion/duy4P5Ga+A8zxnDPnDXMXLSL35/Vk1vO6FHzwruXwIqZtmtZST50Hg0Tn7a9GlTLUFpsD+ZVmzvSNtkxnvIzYdv3kJBkL6YufNo2vUS0h32r7XpDr7LjvqSstBdp134Aw661zTEpK2yyrnzWWF4Oa96DzqOOTPJlpfbAoJqNJpX4RSQBeA14GLhdE/+RyssNd7y7kvd/3sM95/fhV6ccI5Hn7rP/qD/8w/Z7HngJjPotxHTzTsCqYax5z47VcuEL9ne3bzV8fJvtJdNxuO3Z1WeCvVP0u4cOrxcUaS+MAgy81N7+7wmwzTym/OgbnJTPaGqJfzbwCBAB3KGJ/2ilZeX8dtbPfLp6Hw9f2J8rRnQ+9koHdsD3f7c1Oj8P9J0IZz8EodGNHq+qorzMtqcnJB3Z/FZebm8gKj4IK9+CDifZ0Rr3rYQXxta8vfC2tnZfXjFMr9ieNr3H2wunGVuhtMDW7LW5TzmaTOIXkfHAecaYm0RkLDUkfhG5HrgeoFOnTift3LnTq3E2BcWl5dzwxjK+25jK45cM4qKhCXVbMTvZXnRb96Ht9tZusD0IhEbbnhZ6L8DxKy+zB9ae42w/c4+/PePK3gMRbW07+TcPwqJn7fKn32PHX0pZYW82Ki+H1LXVbzssDsb9zXZjTFlp+7Cf9RcYcIntt56bYkdxbdsPwuPtAH9K1aIpJf5HgKuAUiAYiATeN8ZcWdM6vljjr1BYUsb0GUv4aVsGz1w+lHMH1OPCWfIy2+9567eHL/D5+cPoW22TQEyPuo2Tog5bOcuOkQ72Wkrv82HFW/YuUrCDeRXnHrlOQCj0uxA2fmrbySPb2Rr6r762vbQ2fGzHbR97NySOtuukrIKY7hAY6r3vplqcJpP4j9h5LTX+ynw58QPkFZVy9SuLWZWcxfNXncQveret3wbKy+14I8bA/MdsEwPYEQRPmma718UPbLl3NFYe+6U6dRkorCDLtq3PvaVSH3ZHp5H2hqT8TNj5I5z1oL0xyT/I9osfeBnE9YTc/bYbblRHu2xYjB0aeMtX0PcCbYtXDU4TfzOXU1jCFS8uYuO+XJ69cihn9Kln8q8sdYMduXDB05Cx2ZkodgyTooO29hrX24582O9Cm5SaS2+OshJ7QbPooG3aWvehHS+9TR/7sIvOY2zyFj97c1FEW9uscvZDtrkmtgd0Pc2OuLhsBnQcATl77aBeYG9emvAU9DrfTovqCJ1HuvqVlapJk0z8daWJ38rOL+GqVxaxPiWHZy4fytkVY/mfiNz9tv159yJY9a4dCKtVJzDYMVqyd9nlojrZewbiB8BZD9j26GUzbE23Po+FW/GmXae66wybv4av74Or5hzdfl1eDh/91g4jcN5jdn5pkR2Aq+c4e2HzvxdAnjOiYkk+nPFn+N+TRw7uVR+B4VBSYIcpiB9gR3MceFnzOQgqn6eJv4XILijh6lcWs3ZPNk9fPoRx/Rv4ZpmKoXHhcH/uJS/ZM4Q2fSF9k02sFfxD7CBa7Qfb9ursZNv2/Yt7bI3655m2a2LrLvaC55av7bjmt2+wD5rZ8ImtlXc7wz4oozjXPovgwmdh6St23sDLYPU7h0dhFI8driIoErZ+Y2vhIa1sr6Ze59v4Nn9xOMZrv7JnAZ/fBanrbRLP3Aa/WWwPZpu+gL3LYczt9glLC56yB6Zz/gqI/c7RXfXBOKrZ0cTfguQUljDtlcWsTM7mqcuGcP7ARr5TsqzEDqyVMNxeK1g/F3Ytsg/GyNp15MMpupxqh5QuLax9m2Fxdtk17x2eJn52+sH9Ry8vfnDeo7aH0pdOTxkETv8/20aeuR3GPQIDJtnlN3wCsy63zzCe4DzHtLTYHgACgo88wCnVQmnib2EOFpUy7ZXF/Lw7i4cu6M+U4S510azonvjlPbZP+fDrbG168Yu2XT1h+OGBv8rL7F2lke3gh0ftWUSrzs5Qve/Y6woJw2DvCvj2QTt++/Dr7dDUQ68+3OMF7IGn+KAdL6YmzWTMdKUaiyb+FiivqJSbZi7nh01p/PrUrvxxXO+jR/VsqspKYfkM240xrpfb0SjVItWU+LUTdzMWFuTPy1OTuPLkTjw/bxs3zVxOYUnZsVdsCjz+MOxXmvSVcoEm/mbO3+PHgxP7c+/4vnyxbh+/m7WCkrJyt8NSSjVhmvhbABHh2jFduPf8vny+dh9XvbyInRl5boellGqiNPG3INPHdOEfFw9k7d4cJj+/kF0Z+W6HpJRqgjTxtzCTh3Vk9g2jKCotZ/LzC3ln6W63Q1JKNTGa+FugXvERvHHtCGIjArlz9ipeX7jD7ZCUUk2IJv4Wqn+HKObcNJoz+7Thz3PX8vwPW2kOXXeVUo1PE38L5u/x4+nLhzKuXzyPfLaBp7/dQnGp9vhRytdp4m/hggM8/OeKoZzTry2Pf7WJCU//j71ZBW6HpZRykSZ+HyAiPHP5UP49ZQjJBwq44JkfWZ18nCNWKqWaPU38PsLf48cvB7XnvRtHEeDxY/LzC/nP91s4kFfsdmhKKS/TxO9jesVH8MFvRtGnXQT/+Hwjl76wkH3ZxxhJUynVomji90FtIoJ578ZRzLhmGMkHChj/7/ks23lAe/0o5SM08fsoEWFsrzZ8+JvRgHDxswu4Vcf5UconaOL3cT3aRjD35tFcO6YLc1fu5Zwn5vHDpjS3w1JKNSJN/Ir2rUK4d3xfXp5qh+2e+spi/v75huYzxLNSql408atDzujTlk9/ewqTkxJ49vutnP2veby7dDe7M3WwN6VaEk386gjBAR7+MWkQb143gtzCEv4we5U2/yjVwmjiV9Ua1S2W2TeO4oGJ/egUHcp1/13KP7/cyMGiUrdDU0qdIE38qkbd4sK5emQib153Mmf1bctT327hl//+H99tSNWun0o1Y5r41TFFhwXyzOVDmXX9yeQWlnLNjCVMePpHXpq/TQ8ASjVDmvhVnZ3cNYYf7zqde8f3JaewhIc+Wc+4J+Yze1kypdr/X6lmQ5pDjS0pKcksXbrU7TBUJcYY3l2WzGsLdrB2bw5dY8O49cwe/HJge/z8xO3wlFKAiCwzxiRVna41fnVcRITJSR35+JYxPHflSQR4/Lh11grGPTmP7zemuh2eUqoWmvjVCRERxvWP57NbT+GpKUMoLTNcM2MJ02cs4dPVKZSXN/0zSqV8jb/bAaiWwc9PmDCoPWf2acMf3l3Fou0ZfLshlS6xYYztFccNp3WjbWSw22EqpXChjV9EOgL/BdoCBnjBGPNkbetoG3/zU1ZumLtyD+8v38Oi7ZnEhQdxVt+29G0fycVDE/DodQClGl1NbfxuJP52QDtjzHIRiQCWARcYY9bVtI4m/uZtyY5M7npvFckHCigqLad3fATn9IvnN6d3J9BfWxuVaixNJvEfFYDIh8DTxpivalpGE3/LYIzho1UpPPf9Vtal5NA5JpRx/ePpEhPGpJMS8PfoQUCphtQkE7+IJALzgP7GmJwq864Hrgfo1KnTSTt37vR6fKrxfLF2H6/8bzuLtmcC0Ds+gkknJTCkU2uGdmqFiDYFKXWimlziF5Fw4AfgYWPM+7UtqzX+lislu4BvN6Ty3wU72bg/F4BBCVFMHNyBSUkJRAYHuByhUs1Xk0r8IhIAfAx8YYz557GW18Tf8pWVG37cks6GfTm8v3wPG/blEhLgoU1kEOP6xXPbWT0JDvC4HaZSzUqTSfxiz+FfAzKNMb+ryzqa+H3P6uRsZi/bzc7MfL7fmIYInD+gHSO6xtC/fSSDElrpHcJKHUNNid+NfvyjgauA1SKywpn2J2PMpy7EopqoAQlRDEiIwhjDF2v3s3h7Jm8u3snHq1IAiA0P4tYzunPlyZ31eoBS9eR6r5660Bq/Akg/WMT+nEI2pOTy3vJkFmzNoGtsGGXGcNXJnekVH8GY7rF6IFDK0WSaeo6HJn5VlTGGtxbv5p2lu1mxO+vQ9JiwQAZ3bMW94/uSGBvmYoRKuU8Tv2qx1uzJZlt6Hqk5hby5eBfb0vIA6Nc+kvjIYCYMbs+pPeJoHRbocqRKeZcmfuUz9mYVMGvJbj5fk0JKdiG5haWEBno4pUcsrUMD6dc+kgmDOxAVol1FVcumiV/5pKLSMtan5PLagh0s2ZFJXlEpB/JLCPL3Y1hiNDeN7cbIbjF6XUC1SJr4lcJeG1izJ4cPft7Dx6v2kppbRGx4IK1CA+kcHcr0MV0Y3T3W7TCVahCa+JWqorCkjHeX7mbe5nQy84pZvSeb4tJyElqHcHbfeAZ1jOKcfvEEevz0ngHVLGniV+oYsvNLmLVkF4u3ZzJ/czrFznOEQwI8XH9qVyYObo/HT+gco72FVPOgiV+peigsKWPepjQWb89kV2Y+X67bf2jeKT1imTK8Ez3ahNOhdQihgfo8I9U0aeJX6jgZY5i/OZ3t6XnkFJTw+k87Sc0tAkAEzhvQjqGdWjOyawx920e6HK1Sh2niV6qBFJeWs3pPFskHClidnM3bS3aTW1RKgEdI6hxNTHggY7rHsmZvNree0ZO4iCC3Q1Y+ShO/Uo2kuLScXZl5PPH1ZnYfKGDjvhwKS+z1gfAgf8b2iiOhdSg924aTcbCYc/rF0ykm1OWolS/QxK+UlxzIK2Z/biFpuUXM+HEH87ekU1xafmh+VEgAU0d2pkPrENpFhXBKDx1fSDWOpjQ6p1ItWuuwQFqHBdI7Hk7pEUdBcRmpuYUs2XGAhNYh3Dl7Ff/+bgsVda6kzq05WFTK6b3bMDmpI36C9hxSjUpr/Eq5ICW7gAN5JczbnMasxbuIDAlgVXL2ofn92kcypFMresdHMqBDFAADE6L0zEDVizb1KNWEGWOYu3IvJWWGtNwi5m9OY3VyNrlFpYeWuWxYR/q0iyQ4wI9zB7QjIshfDwSqVpr4lWqGNuzLITmzgHmb0/jvwp1HzAv09+Pc/vGc2iOOkEAPbSOD6Nc+Sh9RqQ7RxK9UM2aMYe3eHMA+n/ibDakkZ+bz/s97jliuVWgAnWPC6N02gqAAPwC6xYVz1cmdddgJH6SJX6kWaEtqLn4i5BWVsftAPt9uSGVvVgE/bcsgwONHkdObKDEmlB5tI4iLCOKCwR3o0SZcn0/gAzTxK+VDsvNLCAn0EOAR5qzYw5yf97JmTzYZecWHlgkJ8BAXEURC6xDOG9CO8we0IzzYnwCPHxkHi4gKCcDf4+fit1AnShO/UooN+3LYnVnA1rSDpOUWkZJdwLa0PDbsywUgwGMHoduSepC4iCC6x4VzVt+2nNW3LbmFpXRrE0aQv15DaC408SulqmWM4adtmaxPyWF7eh5b0w4yqGMrftiYRklZOZtTDx5aNtDjx4iu0YzpHkvnmFDatwphb1YBmXklXDS0g15YbmI08Suljsum/bnM25RGTHgg6/bm8NW6/ezIyD9qufZRwYzqHkuQvx/lxjC0U2uMgdE9YokODSQkUA8K3qaJXynVIIwxpB8sZuG2DAqLywgL8icyxJ+nvtnM9vR88opK8feTI+5BEIHEmDBKy8sZ1jmadq2CGd0tlvatQvhuYyr92keR0DqEIH8/osMC9f6EBqKJXynlFeXlhjJj2Jp2kAN5JWxOzWXPgQJWJmcR5O9hS+pB9ucUUlpefe45p19bIoMDCA30EBzgoUPrEDrHhLE/u5CR3WLoGK0D3NWVjtWjlPIKPz/BD6F3vH02wchuMUctcyCvmFV7slmwJZ2+7SMpKzekZBeycncWC7ZmEBroIb+4jMKSMkrKjjxAxIYHEhUSwOCOrQkP8rAnq4DQQH/G9Y/nQH4xp3SPo7S8nKAAD22cIbEDtHfSEbTGr5RqsvKLS0nJLuTb9akM6tiK5bsOsDszn+3ptidSQXEZ7VsFk5JdSH5x2VHrB3jE+elHRLA/naJD2XOggPMGtKNzbBhhgR6yC0oYmNCK0EAPbSODaR0agIhQXFpOgEeadbOT1viVUs1OaKA/3eLC6RYXDsDwLtHVLpeVX8yOjHyC/P1YsDUDf+cu5e3peezKzCcsyJ8gfz827suluKycl/63vcZ9BniE2PAgUrIL6RobRkx4IB1bh9IpJpTOMaF0ig6lTUQwm1NzCQ3056TOrckvLuNgUSntIoObxR3SmviVUs1eq9BABofaO5H7tKv98ZfGGPZkFRDg8WNHeh6B/n5sT88jyN9Dam4h+3IKWbEri25x4ZQ51ysWbsvggxV7qKmBRASMgdBAD+1bhZBfVEr/DlFk5hXTLS6cjLxiElqHEBkSwKZ9ucRG2OaqPu0iiY8MJr+4jNTcIjan5nLx0AR6tAl3tts4BxFt6lFKqTooLCkj+UABuzLzSM0pol2rEDLzitiVUYDBEBsexIZ9OezMyCcyOIDVe7KJDQ9kW3oe4UH+pOUWUVxWTpfYMDIOFpNXVFrtBW4/gbBAfwpLy4gMDuDtX4+ku3MgqC9t6lFKqRMQHOChe5vw407ChSVlFBSXHRojqai0jM37D5KRV0xooIdWIQFEhQbw+sKdZOYVExkSQE5BCa1DAxryawAuJX4RGQc8CXiAl4wxf3MjDqWU8pbgAM8RdzYH+Xvo7zxkp7Lfn92r0WPxeh8nEfEAzwDnAn2BKSLS19txKKWUr3Kjc+twYIsxZpsxphiYBUx0IQ6llPJJbiT+DsDuSp+TnWlHEJHrRWSpiCxNS0vzWnBKKdXSNdnb2YwxLxhjkowxSXFxcW6Ho5RSLYYbiX8P0LHS5wRnmlJKKS9wI/EvAXqISBcRCQQuA+a6EIdSSvkkr3fnNMaUisjNwBfY7pyvGGPWejsOpZTyVa704zfGfAp86sa+lVLK1zWLIRtEJA3YeZyrxwLpDRhOQ9G46qepxgVNNzaNq35aYlydjTFH9Y5pFon/RIjI0urGqnCbxlU/TTUuaLqxaVz140txNdnunEoppRqHJn6llPIxvpD4X3A7gBpoXPXTVOOCphubxlU/PhNXi2/jV0opdSRfqPErpZSqRBO/Ukr5mBad+EVknIhsFJEtInKXy7HsEJHVIrJCRJY606JF5CsR2ez8bO2FOF4RkVQRWVNpWrVxiPWUU36rRGSol+O6X0T2OGW2QkTOqzTvbieujSJyTiPG1VFEvhORdSKyVkRudaa7Wma1xOVqmYlIsIgsFpGVTlx/caZ3EZFFzv7fdoZrQUSCnM9bnPmJXo5rhohsr1Reg53pXvvbd/bnEZGfReRj53PjlpcxpkW+sMNBbAW6AoHASqCvi/HsAGKrTPsHcJfz/i7g716I41RgKLDmWHEA5wGfAQKcDCzyclz3A3dUs2xf5/cZBHRxfs+eRoqrHTDUeR8BbHL272qZ1RKXq2XmfO9w530AsMgph3eAy5zpzwE3Ou9vAp5z3l8GvN1I5VVTXDOASdUs77W/fWd/twNvAh87nxu1vFpyjb85PPBlIvCa8/414ILG3qExZh6QWcc4JgL/NdZPQCsRaefFuGoyEZhljCkyxmwHtmB/340RV4oxZrnzPhdYj31+hKtlVktcNfFKmTnf+6DzMcB5GeAXwGxnetXyqijH2cAZIiJejKsmXvvbF5EE4HzgJeez0Mjl1ZITf50e+OJFBvhSRJaJyPXOtLbGmBTn/T6grTuh1RhHUyjDm51T7VcqNYW5EpdzWj0EW1tsMmVWJS5wucycZosVQCrwFfbsIssYU1rNvg/F5czPBmK8EZcxpqK8HnbK618iElQ1rmpibmhPAHcC5c7nGBq5vFpy4m9qxhhjhmKfNfwbETm18kxjz91c71vbVOJwPAt0AwYDKcDjbgUiIuHAe8DvjDE5lee5WWbVxOV6mRljyowxg7HP2hgO9PZ2DNWpGpeI9AfuxsY3DIgG/ujNmERkPJBqjFnmzf225MTfpB74YozZ4/xMBT7A/kPsrzh9dH6muhReTXG4WobGmP3OP2s58CKHmya8GpeIBGCT60xjzPvOZNfLrLq4mkqZObFkAd8BI7FNJRWjAVfe96G4nPlRQIaX4hrnNJkZY0wR8CreL6/RwAQR2YFtjv4F8CSNXF4tOfE3mQe+iEiYiERUvAfOBtY48Ux1FpsKfOhGfLXEMRe42unhcDKQXal5o9FVaVO9EFtmFXFd5vRw6AL0ABY3UgwCvAysN8b8s9IsV8usprjcLjMRiRORVs77EOAs7PWH74BJzmJVy6uiHCcB3zpnUN6Ia0Olg7dg29Erl1ej/x6NMXcbYxKMMYnYHPWtMeYKGru8GvLKdFN7Ya/Mb8K2Mf6fi3F0xfaoWAmsrYgF2zb3DbAZ+BqI9kIsb2GbAEqwbYfX1hQHtkfDM075rQaSvBzX685+Vzl/8O0qLf9/TlwbgXMbMa4x2GacVcAK53We22VWS1yulhkwEPjZ2f8a4M+V/gcWYy8qvwsEOdODnc9bnPldvRzXt055rQHe4HDPH6/97VeKcSyHe/U0annpkA1KKeVjWnJTj1JKqWpo4ldKKR+jiV8ppXyMJn6llPIxmviVUsrHaOJXLZ6IHHR+JorI5Q287T9V+bygIbevVGPQxK98SSJQr8Rf6e7JmhyR+I0xo+oZk1Jep4lf+ZK/Aac4467f5gza9aiILHEG6fo1gIiMFZH5IjIXWOdMm+MMsLe2YpA9EfkbEOJsb6YzreLsQpxtrxH7HIZLK237exGZLSIbRGRmxeiKIvI3sePrrxKRx7xeOspnHKs2o1RLchd2rPrxAE4CzzbGDHNGZfxRRL50lh0K9Dd2CGOA6caYTOd2/yUi8p4x5i4RudnYgb+qugg7UNogINZZZ54zbwjQD9gL/AiMFpH12CEWehtjTMXwAko1Bq3xK192NnY8lhXYIY1jsGPYACyulPQBfisiK4GfsINk9aB2Y4C3jB0wbT/wA3YEyIptJxs7kNoKbBNUNlAIvCwiFwH5J/ztlKqBJn7lywS4xRgz2Hl1McZU1PjzDi0kMhY4ExhpjBmEHfMl+AT2W1TpfRngb+zY6sOxD9cYD3x+AttXqlaa+JUvycU+prDCF8CNzvDGiEhPZ/TUqqKAA8aYfBHpjX0UX4WSivWrmA9c6lxHiMM+WrLG0TCdcfWjjDGfArdhm4iUahTaxq98ySqgzGmymYEd9zwRWO5cYE2j+sdffg7c4LTDb8Q291R4AVglIsuNHU63wgfYcehXYkfRvNMYs885cFQnAvhQRIKxZyK3H99XVOrYdHROpZTyMdrUo5RSPkYTv1JK+RhN/Eop5WM08SullI/RxK+UUj5GE79SSvkYTfxKKeVj/h/x9MbLJQ8f2QAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MVucuBHML2lT"
      },
      "source": [
        "## Training "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "047e7ea6910640848cbcb155fe296193",
            "1280c1131ab54268ac619ca710053846",
            "02cb620f3c394566b4bba001cf74caa7",
            "108b3f97b4b7448f911aac221f29000a",
            "67f02f1f5ce04b9d852211f691dd93e0",
            "bb591cc65cf04f17a9980a167abbc07c",
            "3640810a345c47c9abb87c0ecfe3c5f5",
            "4d6debce0f174a29a055fff42ec8852b",
            "0aeb482e83f549688e0bba446154fc3f",
            "502edafc3293486784b042dda9ea531d",
            "e1464df5a2ad49a69243f689ed9978da",
            "81a590404169411fafbb4c8591a4d763",
            "f6e031a5948d431d926dc90d0c19bf85",
            "40a65ae6147a46efa0f057d39ae4b315",
            "c270be78409b40c8b0a21c5b5477c0d8",
            "d397713870e94f46bea94cce9c13a7ff",
            "007b2d390dce44b1bfa35d3ae7182dfa",
            "f9f6ac8d669a413eb78bd3c805b7f5f2",
            "e7a545948cc44d7883d12cf7e5056881",
            "2ac0fb04a9f74bc3aa529f1bca0dd607",
            "8e2ae38b3ec14df8b341bee76a87b4a1",
            "e6087a4e884642b38db8fd5342e58dd8",
            "30894eff1bb142cf9a1f631831ac07d4",
            "2d2a783526424fbca96f8225820b86da",
            "49baa3f2f59742fa81a1fabd8981f3c5",
            "00876f42f47e4896989c54bc70184bcc",
            "338c0066444741cabd2165806a0bf651",
            "3da7165548a74127aaa1c2b8228c4fdb",
            "423c2101d7234b8b84293d890005a016",
            "44fcc227c6304583bb95ef69a66e509d",
            "1d4504db12044e10ac1224f6c30fa87a",
            "36ee3099fcee4914b1705ffc18506a49",
            "d7663b6bed184d85a7da2cd78cc0f84c",
            "93ac0c061322415da234443fe260faef",
            "f6baf57cde5a406b8e8240c830f45243",
            "66754e234a504371a518949a691dc702",
            "b688021c3cad4d18b76379c294d0ff7d",
            "cf2a289b7dc044bc97be1938ec312178",
            "b7112ed85c034ae2a4a761af9a895487",
            "e6609b8ef37541acb304c02665661fc7",
            "76e9ed1c22394fb480affa8f4a36e848",
            "75b1ef90e4ee468f81f708ab585d00c5",
            "1219b0624c924dfaa0bba4770a93b121",
            "199ce727311645968992bc431d1d1a1a",
            "958071696fed49ada93937a8f18db3ac",
            "9df0fd35bd21466daf9be6232f80ddf2",
            "24d8dc0461dd456b9733d2603fea59d7",
            "34bb321060644abfba8b199d360dd079",
            "b9dc8d7200af46cbb0af55820281a987",
            "7bd9e79272694ab7ade6212b28abbeae",
            "4f77480efc754c3aa528ff19e89ce6b2",
            "725782c7a532495cb32ae25b24a1454b",
            "14761e84f8ec42b0a307446532bfa7bb",
            "b09c2777f50a490ab63c683de5fb0932",
            "8c5b1b5c24ee45ceb7e8793176305321",
            "ed6d117203b340f79938324a73252313",
            "51c037ab0ecd4d4eb902936170c9ff6e",
            "4a8f8a1ebdfe4ae6835493f32edeb94c",
            "521aca5bb62c487fb1dab1ab913a8398",
            "989b6be9a3894d83a178a819c33311b2",
            "1bdc1fe9780745ba85ab81aa693db85c",
            "613d75963ee4406087a83d8b240b5f03",
            "22034f3cf78e4fc18ba98691cb701163",
            "74a151e3fd9b4daf8d900347c6c4e508",
            "4fd1260290134703ae235b3861127b87",
            "1d661180572b4d3eaed7b9f375ed05d6",
            "ac88991810f043dfbc5d9a0d3f18338e",
            "cc56530c4430435eb11cc194eb8e1970",
            "3c6d5a6abe84471d9a46b55e2a077d3c",
            "e236e58824b74e92a78c20d6aed204dd",
            "221a022ef2be41fcba2c3336e4e78b27",
            "f3e4680c4df9421888b48e68969d80ff",
            "6fa3f6d5957044238727844c94a12985",
            "ea05ebebfb624ab986888c7037198785",
            "f8f052a1e15349b5b4c3cb69c39110a3",
            "67460dec86a549889a41dd56945cb41e",
            "dbf5c651bf7e4a2ba4a126e700acdf2d",
            "a4dfe8f1862a4d1c9f5b791fc73b4a63",
            "1e82fffeb39c4f4ab7a8df93f15664fb",
            "a027725bdfdc4da68ac0c44203c2bb03"
          ]
        },
        "id": "Isn7ue_bL2lT",
        "outputId": "3750e37d-5e8e-4fe0-c7f7-c1a77966bb4a"
      },
      "source": [
        "torch.manual_seed(2045)\n",
        "\n",
        "pad_idx = 0\n",
        "\n",
        "# Parameters\n",
        "params = {'batch_size': 64,\n",
        "          'shuffle': True,\n",
        "          'num_workers':2,\n",
        "          'collate_fn' : BatchPadCollate(pad_idx=pad_idx)\n",
        "          }\n",
        "\n",
        "# Training and validation Data Generators\n",
        "training_set2 = AFEC_Dataset(en_train_dic_,fa_train_dic_,partition2['train'],en_vocab_dic_, fa_vocab_dic_)\n",
        "training_generator2 = torch.utils.data.DataLoader(training_set2, **params)\n",
        "\n",
        "validation_set2 =  AFEC_Dataset(en_train_dic_,fa_train_dic_,partition2['validation'],en_vocab_dic_, fa_vocab_dic_)\n",
        "validation_generator2 = torch.utils.data.DataLoader(validation_set2, **params)\n",
        "\n",
        "# Hyperparameters\n",
        "embed_dim = 256\n",
        "num_heads = 8\n",
        "dim_key = 64\n",
        "dim_value = 64\n",
        "dim_inner = 1024\n",
        "num_layers = 3\n",
        "src_vocab_size = len(en_vocab_dic_.get_str_2_idx())\n",
        "trg_vocab_size = len(fa_vocab_dic_.get_str_2_idx())\n",
        "learning_rate = 1e-4\n",
        "num_iterations = 10000\n",
        "\n",
        "kwargs = {'p_dropout': 0.1,\n",
        "          'kdim': dim_key,\n",
        "          'vdim':dim_value,\n",
        "          }\n",
        "# Model\n",
        "# model2 = Transformer(num_layers, embed_dim, dim_inner, num_heads, src_vocab_size,trg_vocab_size, device,**kwargs).to(device)\n",
        "\n",
        "# Define Loss function and Optimizer\n",
        "criterion = nn.CrossEntropyLoss(ignore_index = pad_idx)\n",
        "optimizer = optim.Adam(model2.parameters(), lr = learning_rate, betas=(0.9, 0.98), eps=1e-9)\n",
        "\n",
        "# train_losses2 = []\n",
        "# validation_losses2 = []\n",
        "\n",
        "torch.autograd.set_detect_anomaly(True)\n",
        "\n",
        "train_dataiter2 = iter(training_generator2)\n",
        "\n",
        "trainloss, valloss, running_trainloss, running_valloss = 0.0, 0.0, 0.0, 0.0\n",
        "\n",
        "# Loop over epochs\n",
        "for itr in tqdm(range(num_iterations)):\n",
        "\n",
        "    # Training        \n",
        "    batch_train_src, batch_train_trg = train_dataiter2.next()\n",
        "\n",
        "    # Transfer to GPU\n",
        "    batch_train_src, batch_train_trg = batch_train_src.to(device), batch_train_trg.to(device)\n",
        "\n",
        "    # zero the parameter gradients\n",
        "    optimizer.zero_grad()\n",
        "    \n",
        "    # Model computations\n",
        "    # forward + backward + optimize\n",
        "    batch_train_outputs = model2(batch_train_src, batch_train_trg[: , :-1])\n",
        "    \n",
        "    loss = criterion(torch.transpose(batch_train_outputs,1,2), batch_train_trg[: , 1:])\n",
        "\n",
        "    loss.backward()\n",
        "\n",
        "    # Clip to avoid exploding gradient issues, makes sure grads are within a healthy range\n",
        "    nn.utils.clip_grad_norm_(model2.parameters(), max_norm=1,error_if_nonfinite= False)\n",
        "\n",
        "    optimizer.step()\n",
        "\n",
        "    # print statistics\n",
        "    trainloss += loss.item()\n",
        "    \n",
        "    # Validation\n",
        "    if itr % 1000 == 999:\n",
        "        with torch.set_grad_enabled(False):\n",
        "            val_cnt = 1\n",
        "            for batch_src, batch_trg in tqdm(validation_generator2):\n",
        "                # Transfer to GPU\n",
        "                batch_src, batch_trg = batch_src.to(device), batch_trg.to(device)\n",
        "                \n",
        "                # Model computations\n",
        "                # forward + backward + optimize\n",
        "                batch_outputs = model2(batch_src, batch_trg[: , :-1])\n",
        "                \n",
        "                val_loss = criterion(torch.transpose(batch_outputs,1,2), batch_trg[: , 1:])\n",
        "\n",
        "                valloss += val_loss.item()\n",
        "                val_cnt += 1\n",
        "\n",
        "        print(\"Cost of Train data after %i iterations : %f\" %(itr + 1, trainloss / 1000))\n",
        "        print(\"Cost of Validation data after %i iterations: %f\" %(itr + 1, valloss / val_cnt))\n",
        "        running_valloss = 0\n",
        "\n",
        "        train_losses2.append(trainloss/1000)\n",
        "        trainloss = 0\n",
        "        validation_losses2.append(valloss/val_cnt)\n",
        "        valloss = 0 "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "047e7ea6910640848cbcb155fe296193",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=10000.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0aeb482e83f549688e0bba446154fc3f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1070.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 1000 iterations : 4.320658\n",
            "Cost of Validation data after 1000 iterations: 4.402590\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "007b2d390dce44b1bfa35d3ae7182dfa",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1070.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 2000 iterations : 4.323213\n",
            "Cost of Validation data after 2000 iterations: 4.393533\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "49baa3f2f59742fa81a1fabd8981f3c5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1070.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 3000 iterations : 4.314763\n",
            "Cost of Validation data after 3000 iterations: 4.381490\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d7663b6bed184d85a7da2cd78cc0f84c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1070.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 4000 iterations : 4.310287\n",
            "Cost of Validation data after 4000 iterations: 4.368315\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "76e9ed1c22394fb480affa8f4a36e848",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1070.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 5000 iterations : 4.306546\n",
            "Cost of Validation data after 5000 iterations: 4.359262\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b9dc8d7200af46cbb0af55820281a987",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1070.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 6000 iterations : 4.291003\n",
            "Cost of Validation data after 6000 iterations: 4.353517\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "51c037ab0ecd4d4eb902936170c9ff6e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1070.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 7000 iterations : 4.296267\n",
            "Cost of Validation data after 7000 iterations: 4.345494\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4fd1260290134703ae235b3861127b87",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1070.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 8000 iterations : 4.284499\n",
            "Cost of Validation data after 8000 iterations: 4.334650\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6fa3f6d5957044238727844c94a12985",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1070.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 9000 iterations : 4.283088\n",
            "Cost of Validation data after 9000 iterations: 4.327861\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "StopIteration",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mStopIteration\u001b[0m                             Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-37-1996b4565c90>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m     \u001b[0;31m# Training\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 55\u001b[0;31m     \u001b[0mbatch_train_src\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_train_trg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_dataiter2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     56\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m     \u001b[0;31m# Transfer to GPU\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    519\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sampler_iter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    520\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 521\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    522\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    523\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1174\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_persistent_workers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1175\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_shutdown_workers\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1176\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1177\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1178\u001b[0m             \u001b[0;31m# Now `self._rcvd_idx` is the batch index we want to fetch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mStopIteration\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "74e961e2c38541b0bca145a64a2ae549",
            "39809a7360d54b1582f1c3b143395157",
            "88dfdfd0740c49d58c4390f9dd161b8a",
            "61b83a5a621e47adaaa78eccc3f7431f",
            "06eb8b809b4741f692bcdc8773ccee1f",
            "7942e610bf7749799d65be3bf9b8279b",
            "01e4afc9f99c45f5979684cfd510fb8d",
            "7d06a4f617ca4c359b6b4d2b23ac36ac",
            "89ac55a41716490787d9fd39185690fe",
            "9eb957c8ac094d318d63ed3a6f5db3ac",
            "82a368cf215b4ac39024f9d54d8f2a53",
            "e8a5521e0f8941b9886ea283815850e7",
            "f4d6f0da456747b48e2a834f07b22858",
            "377dfb7a517a40b5b0390408a95513e4",
            "ea4dfcfb54134caba89b1ffb87d3260d",
            "ff8827224397492ba941c5728fd67e84",
            "c1f919bd8c2c4d33bd80a477f66dde1b",
            "1e6708d4c14d4f529df1f00a20f58c0b",
            "2e4e3142ea314b008f5e088da80b3c18",
            "6500298107c44cf79be69b965a09b165",
            "4e735401f8ea43b3b486c9980e3e94cc",
            "cd5e5c74d4114d62a6c2c24f16522004",
            "0f0fdc0d2f52409f8827cdd64e943f44",
            "359f35b2297541fc8b81886238369cb2",
            "6d0000556d164973801c7befa3f1db01",
            "a0aeef536682459dbad254c7e0542dd2",
            "1a01aa94772d4db8ba51120f58f7b95b",
            "f9538c7c2608409ba3c9d47bd1c5e3c0",
            "c16a5ec9cf784200a5b7953e4f3be6b5",
            "2aaf7cb63da0416383cd748228123586",
            "10de73bf343b459998b720dc2440f0e3",
            "a1a4a08ce5774bbdae68f94f9d0327e5",
            "0c8df34ace1441e0accb3ce00eecb291",
            "1db2500183ad42f390e6dce43d4b054a",
            "723b5f39f94049a89a137bf61806ae99",
            "284575d407f04b979fc1654277208ff0",
            "35ee66f5a09048499d8eca3e968cbeab",
            "2d1c68a6510748dda57661bb555c55b1",
            "2dde0eed7b1d48238b9c4c1becf6691e",
            "385f441045484ec7adc2a1f4dbe81572",
            "90ef7acd0ed949ee9886b22130ab7299",
            "513683a4e8a84b009ac0df456124505c",
            "38ede7be101548f8b509828469534e3a",
            "d36cd935e74b402588125c5f49a9c8bd",
            "1aa1633fc153426c8fdb0fc324459cf2",
            "73070231db924cc78e0a0574780c4368",
            "82a67176c4614bf4941a2344907afa12",
            "e12ee6b1e2594c518bc2d4c4ad1cccef",
            "d721b970477949a6865694d8c663bfc5",
            "024dfd2d6c5141f99cbb4777402b5e43",
            "697374da22424d6a8b96c5cb256b1f8c",
            "de728408093447adac068ffd5e680456",
            "21e818fdda224070b0d58fe12f67c432",
            "9756d93d0ddf46faa5dbb04faebcf08f",
            "6fb6610b3aba42c59691cdea25070136",
            "cb0007ef3f96407b9dfc51b8d4ce1c4f",
            "edf62de4cb7d4cfba46b286cddbd0837",
            "dc58a2f86f8d4c57a422ed5a8097e248",
            "6fcb6a66fb394d35aa9a7f0886fe283f",
            "54d26fa22b0f4572944473a83a2df693",
            "11efdae88e7049c18a6554d87cfd7630",
            "c834a18057c746f2abdf5294a0ae1dd1",
            "4d0e557f5f52447fb449242e232c2e59",
            "b9107ba238704327bd27ead475d4e7f7",
            "8df2bd8d5bd3424c89c1899b20c7b0db",
            "1e30548d33af487099f4c3e39f52b4ab",
            "c31b31808e56445a92f8d6b3b2f2164e",
            "027faf164fcf42f68e6905660caea2aa",
            "3485e955d43f452bab1491012045dcb4",
            "088751e8dcb449ca859eeffd2b64581b",
            "0413bc02f8d84ea2872fb1970ed16b49",
            "7f08b599a1dd483f8f53bbc3727ed486",
            "20ff693e87b1484aac68a38f8520f32b",
            "0e997f36b33542299e405d141859a911",
            "331fd6e50c8d4906b814a4de167fb521",
            "1632a71640af45ce9edfd275b1503d32",
            "042c0e36fb1b42a3a1272a7d2fb55c98",
            "d73f9128bc7047eb9b8216bf2d92a88c",
            "1ac58948d89e42c398b592d29367f1f0",
            "094bf8dc763c4405ac9ef4e09aaf6217"
          ]
        },
        "id": "WlRDtL7a5ukY",
        "outputId": "0e664b2f-e419-4270-a1d1-fc28627f9490"
      },
      "source": [
        "train_dataiter2 = iter(training_generator2)\n",
        "\n",
        "trainloss, valloss, running_trainloss, running_valloss = 0.0, 0.0, 0.0, 0.0\n",
        "\n",
        "# Loop over epochs\n",
        "for itr in tqdm(range(num_iterations)):\n",
        "\n",
        "    # Training        \n",
        "    batch_train_src, batch_train_trg = train_dataiter2.next()\n",
        "\n",
        "    # Transfer to GPU\n",
        "    batch_train_src, batch_train_trg = batch_train_src.to(device), batch_train_trg.to(device)\n",
        "\n",
        "    # zero the parameter gradients\n",
        "    optimizer.zero_grad()\n",
        "    \n",
        "    # Model computations\n",
        "    # forward + backward + optimize\n",
        "    batch_train_outputs = model2(batch_train_src, batch_train_trg[: , :-1])\n",
        "    \n",
        "    loss = criterion(torch.transpose(batch_train_outputs,1,2), batch_train_trg[: , 1:])\n",
        "\n",
        "    loss.backward()\n",
        "\n",
        "    # Clip to avoid exploding gradient issues, makes sure grads are within a healthy range\n",
        "    nn.utils.clip_grad_norm_(model2.parameters(), max_norm=1,error_if_nonfinite= False)\n",
        "\n",
        "    optimizer.step()\n",
        "\n",
        "    # print statistics\n",
        "    trainloss += loss.item()\n",
        "    \n",
        "    # Validation\n",
        "    if itr % 1000 == 999:\n",
        "        with torch.set_grad_enabled(False):\n",
        "            val_cnt = 1\n",
        "            for batch_src, batch_trg in tqdm(validation_generator2):\n",
        "                # Transfer to GPU\n",
        "                batch_src, batch_trg = batch_src.to(device), batch_trg.to(device)\n",
        "                \n",
        "                # Model computations\n",
        "                # forward + backward + optimize\n",
        "                batch_outputs = model2(batch_src, batch_trg[: , :-1])\n",
        "                \n",
        "                val_loss = criterion(torch.transpose(batch_outputs,1,2), batch_trg[: , 1:])\n",
        "\n",
        "                valloss += val_loss.item()\n",
        "                val_cnt += 1\n",
        "\n",
        "        print(\"Cost of Train data after %i iterations : %f\" %(itr + 1, trainloss / 1000))\n",
        "        print(\"Cost of Validation data after %i iterations: %f\" %(itr + 1, valloss / val_cnt))\n",
        "        running_valloss = 0\n",
        "\n",
        "        train_losses2.append(trainloss/1000)\n",
        "        trainloss = 0\n",
        "        validation_losses2.append(valloss/val_cnt)\n",
        "        valloss = 0 "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "74e961e2c38541b0bca145a64a2ae549",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=10000.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "89ac55a41716490787d9fd39185690fe",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1070.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 1000 iterations : 4.420891\n",
            "Cost of Validation data after 1000 iterations: 4.493173\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c1f919bd8c2c4d33bd80a477f66dde1b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1070.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 2000 iterations : 4.415193\n",
            "Cost of Validation data after 2000 iterations: 4.478881\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6d0000556d164973801c7befa3f1db01",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1070.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 3000 iterations : 4.413903\n",
            "Cost of Validation data after 3000 iterations: 4.464949\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0c8df34ace1441e0accb3ce00eecb291",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1070.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 4000 iterations : 4.404896\n",
            "Cost of Validation data after 4000 iterations: 4.452687\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "90ef7acd0ed949ee9886b22130ab7299",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1070.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 5000 iterations : 4.390913\n",
            "Cost of Validation data after 5000 iterations: 4.444065\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d721b970477949a6865694d8c663bfc5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1070.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 6000 iterations : 4.386796\n",
            "Cost of Validation data after 6000 iterations: 4.434957\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "edf62de4cb7d4cfba46b286cddbd0837",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1070.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 7000 iterations : 4.376458\n",
            "Cost of Validation data after 7000 iterations: 4.422238\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8df2bd8d5bd3424c89c1899b20c7b0db",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1070.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 8000 iterations : 4.369233\n",
            "Cost of Validation data after 8000 iterations: 4.416033\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "20ff693e87b1484aac68a38f8520f32b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1070.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 9000 iterations : 4.356462\n",
            "Cost of Validation data after 9000 iterations: 4.399969\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "StopIteration",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mStopIteration\u001b[0m                             Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-61-51858e0b2989>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0;31m# Training\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m     \u001b[0mbatch_train_src\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_train_trg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_dataiter2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0;31m# Transfer to GPU\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    519\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sampler_iter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    520\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 521\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    522\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    523\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1174\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_persistent_workers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1175\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_shutdown_workers\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1176\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1177\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1178\u001b[0m             \u001b[0;31m# Now `self._rcvd_idx` is the batch index we want to fetch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mStopIteration\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OZxjA9YiL2lV",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "outputId": "dc741331-ab07-44de-f57c-6e75aee6ee97"
      },
      "source": [
        "# plot the cost\n",
        "plt.plot(range(500,54500,1000),train_losses2)\n",
        "plt.plot(range(500,54500,1000),validation_losses2)\n",
        "plt.title(\"Cost of Transfromer vs iters \")\n",
        "plt.ylabel('Cost')\n",
        "plt.xlabel('Iterations')\n",
        "plt.legend(labels = [\"Train\",\"Validation\"])\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU5fX48c+Z7PvOlgAB2VQii0FFQECtK4q7olXRVotWq7WttXZx+9pFbX9uVWu1Wq2KFhU33LAiuCEB2RfZgmQBkpCV7Mn5/XFvYIhJCJjJJJnzfr3ua+7cbc4zmcyZ53nufa6oKsYYYwKXx98BGGOM8S9LBMYYE+AsERhjTICzRGCMMQHOEoExxgQ4SwTGGBPgLBGYLkdEJojIRhGpEJFz/B1PcyLSW0QWiki5iPzV3/F0BhEZ4P49gvwdi+l4lggCnIhcKiJZ7j95voi8KyITv+cxs0Xk5O9xiLuBR1U1WlXnNjt2hdfUKCJVXs8v+z5xH4RrgUIgVlV/0Umv6Veq+q3792gAEJEFIvJjf8dlOoYlggAmIrcADwJ/BHoDA4DHgOn+jAsYCKxpaYX7ZRStqtHAt8BZXsteaNpORIJ9HN9abeVqTB+/dpv8+doHw2oWXYyq2hSAExAHVAAXtrFNGE6iyHOnB4Ewd10y8DZQAuwGFuH8sHgeaASq3OPf2sqxrwE2ufu+CfRzl29utn9YG/FlAye781OAHODXwA43jgQ3xgKg2J1P89p/AXAP8BlQDnwAJLvrwoH/AEVuGZfgJMtngTqg1o3vZOBOYI67fRnwY6CfW67dbjmv8XrdO4H/utuXA6uAYcBvgF3AduCUZn+rp4F8IBf4PyDIXTfTjf//ubH+X7P3qJ/7XiZ6LRuDU6MJAYYAnwCl7rKXW3mv0wEFgoF7gQag2n0PHnW3GQF86JZ5A3CR1/7PAo8D84A97vt2BrDWfQ9ygV/6+/8iUCe/B2CTn/7wcBpQDwS3sc3dwJdALyAF+By4x133J+AJ98skBJgEiLsuG/cLupXjnuh+6YzFSTaPAAu91re5f0vb4SSCeuAv7jEjgCTgfCASiHG/fOd67b8AJ/EMc7dfAPzZXfcT4C133yDgaJymoKYvtf/zOs6dOMnhHJxkGAEsxKldhQOjcZLRiV7bVwOnul+szwFbgd+67+U1wFav478O/AOIcv8WXwE/cdfNdMt9o3usiBbep/+xfyK6H3jCnX/JfV2PG+vEVt7rdNxE4PXe/dhrfRROArvKjaMp2Rzh9Z6VAhO8XisfmOSuTwDG+vv/IlAnaxoKXElAoarWt7HNZcDdqrpLVQuAu4DL3XV1QF9goKrWqeoidf+j2+Ey4F+qukxVa3B+CY8XkfRDKYiXRuAOVa1R1SpVLVLVV1W1UlXLcX7JTm62zzOq+o2qVgGv4HxpN5UvCRiiqg2qulRVy9p47S9Uda6qNuLUliYAv1bValVdDjwFXOG1/SJVfd99//+Lk2j/rKp1wGwgXUTiRaQ3zi/nm1V1j6ruwvn1f4nXsfJU9RFVrXfL0dyLwAwAERF33xe9yjkQp0ZWraqftlHGtkwDslX1GTeOr4FXgQu9tnlDVT9T1UZVrXZf+wgRiVXVYlVddoivbb4nSwSBqwhIPkCbcj9gm9fzbe4ycH5VbgI+EJEtInLbQbz2fsdV1Qo3ntSDOEZLCtwvGABEJFJE/iEi20SkDOdXenyz9ukdXvOVQLQ7/zzwPjBbRPJE5D4RCWnjtbd7zfcDdrvJp8k29i/fTq/5Kpyk3OD1HDeWgTi1hHwRKRGREpzaQa9WXrslr+Ik2r7ACTgJc5G77lZAgK9EZI2IXH2AY7VmIHBsU4xunJcBfdqI83ycJLdNRD4RkfGH+Nrme7JEELi+AGpwmjNak4fzD95kgLsMVS1X1V+o6mDgbOAWETnJ3e5ANYP9jisiUTi/vnMPqgTf1fx1fwEMB45V1VicL0FwvvjaPpBTy7lLVY8Ajsf5xXtFW7t4zecBiSIS47VsAIdWvu04f6dkVY13p1hVPbKV1/5uYKrFOP0fFwOXArObam+qukNVr1HVfjjNYY+JyJB2xNX8NbcDn3jFGK9OB/51re2jqktUdTpOUpuLUyMzfmCJIECpainwB+DvInKO++s5REROF5H73M1eAn4nIikikuxu/x8AEZkmIkPcpoZSnM7DRne/ncDgNl7+JeAqERktImE4Zy0tVtXsDi5mDM6v6xIRSQTuaO+OIjJVRDLc2kMZTjNG4wF2A0BVt+P0p/xJRMJF5CjgR7jv3cFQ1XycL/G/ikisiHhE5DARad7EdSAv4iSyC9jXLISIXCgiae7TYpwv6/aUs/nf+G1gmIhc7n6OQkRknIgc3tLOIhIqIpeJSJzbHFbWztc1PmCJIICp6l+BW4Df4XRmbgduwPl1Bs7ZKVnASpwzW5a5ywCGAvNxzhr5AnhMVT921/0JJ4GUiMgvW3jd+cDvcZos8oHD2L/Nu6M8iNNxW4jT6f3eQezbB+dMoDJgHc6ZNc8fxP4zcDpY83A6e+9wy30orgBCcc6wKXbj6nuQx3gT52+2Q1VXeC0fBywWkQp3m5tUdUs7jvcQcIGIFIvIw24z2Ck4f8c8nCa3po771lwOZLvNdrNwmpKMHzSd5WGMMSZAWY3AGGMCnCUCY4wJcJYIjDEmwFkiMMaYANctBqjylpycrOnp6f4OwxhjupWlS5cWqmpKS+u6XSJIT08nKyvL32EYY0y3IiLbWltnTUPGGBPgLBEYY0yAs0RgjDEBrtv1ERhjeo66ujpycnKorq4+8MamXcLDw0lLSyMkpK3BcvdnicAY4zc5OTnExMSQnp6OM36h+T5UlaKiInJychg0aFC797OmIWOM31RXV5OUlGRJoIOICElJSQddw7JEYIzxK0sCHetQ3s+ASQTrd5Rx33vrKa2s83coxhjTpfgsEYjIcBFZ7jWVicjNzbYREXlYRDaJyEoRGeureL4tquSxBZv5dnelr17CGNPNFBUVMXr0aEaPHk2fPn1ITU3d+7y2trbNfbOysvjZz37WSZH6ls86i1V1A+6NwN27POXi3KDD2+k4N8sYChwLPO4+drh+8REA5JZUkZEW54uXMMZ0M0lJSSxfvhyAO++8k+joaH75y333Uqqvryc4uOWvyczMTDIzMzslTl/rrKahk4DNqtr8EufpwHPq+BLnxuIHe+eldkl1E0FeSdUBtjTGBLKZM2cya9Ysjj32WG699Va++uorxo8fz5gxYzj++OPZsGEDAAsWLGDatGmAk0SuvvpqpkyZwuDBg3n44Yf9WYSD1lmnj16Cc5/a5lJxbo/YJMddlu+9kYhcC1wLMGDAgEMKID4yhIiQIEsExnRRd721hrV5ZR16zCP6xXLHWUce9H45OTl8/vnnBAUFUVZWxqJFiwgODmb+/PncfvvtvPrqq9/ZZ/369Xz88ceUl5czfPhwrrvuuoM6l9+ffJ4IRCQUOBv4zaEeQ1WfBJ4EyMzMPKR7a4oI/eLDySu1RGCMaduFF15IUFAQAKWlpVx55ZVs3LgREaGuruUTTs4880zCwsIICwujV69e7Ny5k7S0tM4M+5B1Ro3gdGCZqu5sYV0u0N/reZq7zCf6xUeQW2JXMBrTFR3KL3dfiYqK2jv/+9//nqlTp/L666+TnZ3NlClTWtwnLCxs73xQUBD19fW+DrPDdEYfwQxabhYCeBO4wj176DigVFXzW9n2e0uNj7CmIWPMQSktLSU1NRWAZ5991r/B+IhPE4GIRAE/AF7zWjZLRGa5T+cBW4BNwD+B630ZT7/4CArKa6ipb/DlyxhjepBbb72V3/zmN4wZM6Zb/co/GKJ6SE3ufpOZmamHemOaOUtz+OV/V/DJr6YwMCnqwDsYY3xq3bp1HH744f4Oo8dp6X0VkaWq2uL5rgFzZTFAv/hwwLmWwBhjjCOgEsG+awmsw9gYY5oEVCLoE+fUCKzD2Bhj9gmoRBAWHERKTJglAmOM8RJQiQCgX1y49REYY4yXwEsEdi2BMcbsJ0ATQTXd7bRZY0zHmzp1Ku+///5+yx588EGuu+66FrefMmUKTaevn3HGGZSUlHxnmzvvvJMHHnigzdedO3cua9eu3fv8D3/4A/Pnzz/Y8DtMQCaCqroGSuwGNcYEvBkzZjB79uz9ls2ePZsZM2YccN958+YRHx9/SK/bPBHcfffdnHzyyYd0rI4QcIkg1a4lMMa4LrjgAt555529N6HJzs4mLy+Pl156iczMTI488kjuuOOOFvdNT0+nsLAQgHvvvZdhw4YxceLEvcNUA/zzn/9k3LhxjBo1ivPPP5/Kyko+//xz3nzzTX71q18xevRoNm/ezMyZM5kzZw4AH330EWPGjCEjI4Orr76ampqava93xx13MHbsWDIyMli/fn2HvQ+dNQx1l9HP674EI1PtBjXGdBnv3gY7VnXsMftkwOl/bnV1YmIixxxzDO+++y7Tp09n9uzZXHTRRdx+++0kJibS0NDASSedxMqVKznqqKNaPMbSpUuZPXs2y5cvp76+nrFjx3L00UcDcN5553HNNdcA8Lvf/Y6nn36aG2+8kbPPPptp06ZxwQUX7Hes6upqZs6cyUcffcSwYcO44oorePzxx7n5ZufmjsnJySxbtozHHnuMBx54gKeeeqoj3qXAqxH0sxvUGGO8eDcPNTULvfLKK4wdO5YxY8awZs2a/Zpxmlu0aBHnnnsukZGRxMbGcvbZZ+9dt3r1aiZNmkRGRgYvvPACa9asaTOWDRs2MGjQIIYNGwbAlVdeycKFC/euP++88wA4+uijyc7OPtQif0fA1QiSokIJDfaQV2pXFxvTpbTxy92Xpk+fzs9//nOWLVtGZWUliYmJPPDAAyxZsoSEhARmzpxJdfWhfV/MnDmTuXPnMmrUKJ599lkWLFjwvWJtGuq6o4e5DrgagYiQGh9hfQTGGACio6OZOnUqV199NTNmzKCsrIyoqCji4uLYuXMn7777bpv7n3DCCcydO5eqqirKy8t566239q4rLy+nb9++1NXV8cILL+xdHhMTQ3l5+XeONXz4cLKzs9m0aRMAzz//PJMnT+6gkrYu4BIBOIPPWdOQMabJjBkzWLFiBTNmzGDUqFGMGTOGESNGcOmllzJhwoQ29x07diwXX3wxo0aN4vTTT2fcuHF7191zzz0ce+yxTJgwgREjRuxdfskll3D//fczZswYNm/evHd5eHg4zzzzDBdeeCEZGRl4PB5mzZqFrwXUMNRNfvXfFSzcWMDi2/13upYxxoah9hUbhrod+sVHsKu8htr6Rn+HYowxfheQiSA1PgJV2FlmHcbGGBOQiaDpFFLrMDbG/7pb83RXdyjvZ4AmArsvgTFdQXh4OEVFRZYMOoiqUlRURHh4+EHtF3DXEYBdVGZMV5GWlkZOTg4FBQX+DqXHCA8PJy0t7aD2CchEEB4SRFJUKLl2y0pj/CokJIRBgwb5O4yAF5BNQ2D3JTDGmCYBnAjsojJjjIGATgROjcA6qYwxgS5gE0FqfAR7ahsoq+q4gZuMMaY7CpxEUF0Gm+ZDg/PFb9cSGGOMI3ASwTfvw3/Oh4J1gJ1CaowxTXyaCEQkXkTmiMh6EVknIuObrZ8iIqUistyd/uCzYNLcsZZynAHr9l5UVmqJwBgT2Hx9HcFDwHuqeoGIhAKRLWyzSFWn+TgOSEiHyCQnEWReRXJUGKFBHmsaMsYEPJ8lAhGJA04AZgKoai1Q66vXa0dAkDYOcpYA4PEIfePDybOLyowxAc6XTUODgALgGRH5WkSeEpGoFrYbLyIrRORdETmypQOJyLUikiUiWd/rUvS0TCjcAFUlAPSLs4vKjDHGl4kgGBgLPK6qY4A9wG3NtlkGDFTVUcAjwNyWDqSqT6pqpqpmpqSkHHpEae6dg3KXAnZ1sTHGgG8TQQ6Qo6qL3edzcBLDXqpapqoV7vw8IEREkn0WUb+xgOxNBKnx4ewsq6auwW5QY4wJXD5LBKq6A9guIsPdRScBa723EZE+IiLu/DFuPEW+ionwWEgZsbefoF98BI12gxpjTIDz9VlDNwIvuGcMbQGuEpFZAKr6BHABcJ2I1ANVwCXq6zEf0jJh/duguvdagvzSatISWjqhyRhjej6fJgJVXQ40v1nyE17rHwUe9WUM35E2Dr5+HnZvoV98b8AuKjPGBLbAubK4SVOHcU7W3ovK7FoCY0wgC7xEkDIcQmMgZwmRocHER4ZYjcAYE9ACLxF4giB1zL4O47gIu6jMGBPQAi8RgNM8tHM11FbatQTGmIAXuImgsR7yVzAoOZIthXuoqm3wd1TGGOMXgZkIUt0TmXKzmDg0hdr6RhZv9d3lC8YY05UFZiKIToH4gZCzhGMHJRIW7OGTb77HGEbGGNONBWYiAHck0izCQ4I4bnCSJQJjTMAK7ERQlgtleUwelsKWgj1s313p76iMMabTBXYiAMjJYvJwZ0RTqxUYYwJR4CaCPhkQFAY5SxicHEVaQoQlAmNMQArcRBAcCn2PgpwsRITJw1L4fFMhtfU2JLUxJrAEbiIAp3ko72toqGPysBT21DawdFuxv6MyxphOFeCJIBPqq2DnGo4fkkywR6x5yBgTcAI8ETTdujKL6LBgMtMTLBEYYwJOYCeCuP4Q3RtysgCYMrwX6/LL7I5lxpiAEtiJQMQZbuLbLwGYPMxOIzXGBJ7ATgQAh02F4q2wcy0j+sTQKybMEoExJqBYIjjiHBAPrH5172mkn24spL7BTiM1xgQGSwTRKTBoMqx5DVSZPDyF0qo6VuSU+jsyY4zpFJYIAEaeD7u3QP5yJg5JxiPWT2CMCRyWCAAOnwaeEFj9KvGRoYzuH2+JwBgTMCwRAEQkwJCTYPXr0NjI5GG9WJlTwu49tf6OzBhjfM4SQZOR50NZDuR8xeThKajCoo1WKzDG9HyWCJoMPx2Cw2H1q2SkxpEQGcJH63b5OypjjPE5SwRNwmJg2KmwZi5BNHJGRl/eX7OD0qo6f0dmjDE+ZYnA28jzYc8uyP6US8YNoKa+kTeX5/o7KmOM8SmfJgIRiReROSKyXkTWicj4ZutFRB4WkU0islJExvoyngMaegqERsPqVxmZGssRfWN5OWu7X0Myxhhf83WN4CHgPVUdAYwC1jVbfzow1J2uBR73cTxtC4mA4WfAujeRhjouHtef1bllrM61i8uMMT2XzxKBiMQBJwBPA6hqraqWNNtsOvCcOr4E4kWkr69iapeR50NVMWxZwDmjUwkN9vCK1QqMMT2YL2sEg4AC4BkR+VpEnhKRqGbbpALe37I57rL9iMi1IpIlIlkFBT4+pfOwEyE8Dla/SlxkCKeP7MPrX+dSXdfg29c1xhg/8WUiCAbGAo+r6hhgD3DboRxIVZ9U1UxVzUxJSenIGL8rOBQOPxvWvwN1VVw8rj/l1fW8t3qHb1/XGGP8xJeJIAfIUdXF7vM5OInBWy7Q3+t5mrvMv0aeB7XlsPFDjhuUxMCkSGYv+dbfURljjE/4LBGo6g5gu4gMdxedBKxtttmbwBXu2UPHAaWqmu+rmNot/QSISoEVL+HxCBdl9ufLLbvJLtzj78iMMabD+fqsoRuBF0RkJTAa+KOIzBKRWe76ecAWYBPwT+B6H8fTPkHBcPRVsOFdKPiGC45OwyNYp7ExpkfyaSJQ1eVu2/5RqnqOqhar6hOq+oS7XlX1p6p6mKpmqGqWL+M5KMdcC8Fh8PlD9I4NZ+rwXsxZmmM3rDHG9Dh2ZXFrolNgzA9hxctQls/F4/qzq7yGBRtsIDpjTM9iiaAt428AbYAvH2PqiF6kxITZlcbGmB7HEkFbEgc59zTOeoaQ2jLOH5vG/9bvYldZtb8jM8aYDmOJ4EAm3uycSrr0GS4e15+GRuXfX2T7OypjjOkwlggOpO8oGDwVvnycQXFBTDuqL898lk1BeY2/IzPGmA5hiaA9JtwEFTth5Wxu+cEwauob+fvHm/wdlTHGdAhLBO0xeIpTM/jsYQYnRXDh0Wm8uPhbcoor/R2ZMcZ8b5YI2kMEJtwMuzfD+nf42UlDQeCh+Rv9HZkxxnxvlgja6/CzISEdPnuQfnHhXH7cQF5dlsOmXRX+jswYY74XSwTtFRQMx98IuUth60Kun3IYESFB/O3DDf6OzBhjvpd2JQIReb49y3q80ZdBTD+YfydJkSH8aNJg5q3awaocu4OZMab7am+N4EjvJyISBBzd8eF0cSERcOLvIG8ZrHmNayYNIj4yhPs/sFqBMab7ajMRiMhvRKQcOEpEytypHNgFvNEpEXY1o2ZAnwyYfxcxQQ1cP+UwFn5TwJdbivwdmTHGHJI2E4Gq/klVY4D7VTXWnWJUNUlVf9NJMXYtHg+cci+UfguLn+CK8en0jg3j/vc3oKr+js4YYw5ae5uG3m6637CI/FBE/iYiA30YV9c2eDIMPRUW/ZXw2mJuOmkYS7cV8/4au52lMab7aW8ieByoFJFRwC+AzcBzPouqOzjlHqjdA5/8hYsy0xjeO4Z7562zm9wbY7qd9iaCenXaPaYDj6rq34EY34XVDaQMh6OvhKx/EVy8md9PO4Ltu6v412db/R2ZMcYclPYmgnIR+Q1wOfCOiHiAEN+F1U1MuR2CI+DDO5g4NJmTD+/N3/+3yYapNsZ0K+1NBBcDNcDV7k3p04D7fRZVdxGdApN+DhvegexP+e2Zh1Pb0Mj979vppMaY7qNdicD98n8BiBORaUC1qgZ2H0GT466H2DR4/3YGJUZw1YRBzFmWYxeZGWO6jfZeWXwR8BVwIXARsFhELvBlYN1GSAScfCfkr4AvHuGGE4eQGBnK3W+vsdNJjTHdQnubhn4LjFPVK1X1CuAY4Pe+C6ubybgARkyDj+4hdvcafnnqcJZkF/POqnx/R2aMMQfU3kTgUdVdXs+LDmLfnk8Ezn4EolLg1R9x0VGJHN43lj/NW2+nkxpjurz2fpm/JyLvi8hMEZkJvAPM811Y3VBkIpz3DyjaTNAHt3PHWUeQW1LFPxdu8XdkxhjTpgONNTRERCao6q+AfwBHudMXwJOdEF/3MugE57aWy/7NcTWfc0ZGHx75eBNr88r8HZkxxrTqQDWCB4EyAFV9TVVvUdVbgNfddaa5qb+FvqPhzRu598QkEiJD+OmLy6ioqfd3ZMYY06IDJYLeqrqq+UJ3WbpPIurugkPh/KehvoaED27k4YtHsa1oD7e/tsrOIjLGdEkHSgTxbayLONDBRSRbRFaJyHIRyWph/RQRKXXXLxeRPxzomN1C8hA4/S+wdSHH5v+HW34wjDdX5DF7yXZ/R2aMMd9xoESQJSLXNF8oIj8GlrbzNaaq6mhVzWxl/SJ3/WhVvbudx+z6xlwOR0yHj+7h+vR8Jg1N5s4317Au3/oLjDFdy4ESwc3AVSKyQET+6k6fAD8CbvJ9eN2YCJz9KCQdhmfOTB46LYm4CKe/YI/1FxhjupAD3Zhmp6oeD9wFZLvTXao63h124kAU+EBElorIta1sM15EVojIuyJyZEsbiMi1IpIlIlkFBQXteNkuIjwWLnkJGupIfOsqHrlgBNmFe/jd3NXWX2CM6TLEl19IIpKqqrki0gv4ELhRVRd6rY8FGlW1QkTOAB5S1aFtHTMzM1Ozsr7T3dC1ffM+vHgxZFzIw3G/4m/zN/LHczO49NgB/o7MGBMgRGRpa030Pr06WFVz3cddOKecHtNsfZmqVrjz84AQEUn2ZUx+MexUOPG3sOoVboh4nxOGpfD7N1bz8YZdB97XGGN8zGeJQESiRCSmaR44BVjdbJs+IiLu/DFuPD3zLvCTfgmHn41n/h/4x/FlDO8dw09fWMbKnBJ/R2aMCXC+rBH0Bj4VkRU4I5e+o6rvicgsEZnlbnMBsNrd5mHgEu2pjecicM7jkDyciDd+zHPnppAQGcrVzy7h26JKf0dnjAlgPu0j8IVu2UfgrWgz/HMqRKWw9az/cu5zm0iIDGXOrPEkRYf5OzpjTA/ltz4C04Kkw+DSV6Asj0HzLuPZiweTV1LFj/6dRVWtjVRqjOl8lgj8YcBxMGM2FG1m9IKreez8w1iZU8INLy6jvqHR39EZYwKMJQJ/GTwZLv4P7FzLSUt/yr1nDuaj9bv4yfNLbYA6Y0ynskTgT8NOgQv+BblLmbHpV/zxrCEs+KaAC5/4gvzSKn9HZ4wJEJYI/O2Is+Hcf0D2p1y69Xb+9cMMtu+uZPqjn7Eqp9Tf0RljAoAlgq7gqAudW11ums/kJdfz+tUjCQnycNE/vuD9Ne0ZycMYYw6dJYKuYuzlTs1g2+cMnXcRb1w5mGF9Ypj1n6V2u0tjjE9ZIuhKRl0Cl/0XirNJnj2NV85L4PSRfbh33jobjsIY4zOWCLqaw06Eq+ZBfQ1h/z6dB8dXMzg5irvfWktNvV1nYIzpeJYIuqK+o+DHH0JUMqEvnMsjY3LYWriHpz/d6u/IjDE9kCWCriohHa7+APoexZGf3sjN6dt55KNNdlqpMabDWSLoyqKS4PK5kHI4Nxb/iX66g3vfWefvqIwxPYwlgq4uLBou+Q9BosyOfZSPVm7l882F/o7KGNODWCLoDhIHw/n/IrlyEw9HPcOdb6ymzsYkMsZ0EEsE3cXQk5ETf8cPGhYxqegVnv9im78jMsb0EJYIupNJv0APP4vfhrzEZx++RkF5jb8jMsb0AJYIuhMR5JzHqU84jPv5G//vv/OprrNrC4wx348lgu4mLIbQy2YTFQw/zr6F3z74JGvybHA6Y8yhs0TQHSUPIezyV+gXE8Rf99zGmieu5F8fLqWhsXvddtQY0zVYIuiu0icQftMSqsfdwPmeT5j+6XQee+gethft8XdkxphuxhJBdxYaRfiZ9+KZtRASB3Nj6V/Je/gHvPHRJ1Y7MMa0myWCHkD6ZJB04wKKp/6ZkZ5sTlt4Pi8+8DOWbyvwd2jGmG7AEkFP4fGQMPk6Im9ZRmHqiVxe+RxhT5/I3//zMsV7av0dnTGmC7NE0MNITB9Sr32FqvOeIzWsklkbf8Jb91/Fy5+tt+YiY0yLLBH0UBFHTSf2F8soP/KHXMHbHP/+NO687z7e+DrHEoIxZj+WCHqy8GI03S8AABfrSURBVDjiL3oUnTmP+Lg47qn+I4mvXcy1DzzHG8tzLSEYYwAfJwIRyRaRVSKyXESyWlgvIvKwiGwSkZUiMtaX8QQqSZ9AzM1f0njaXzgu/FuerLyZkjk3ceHf3ubNFXk0WkIwJqB1Ro1gqqqOVtXMFtadDgx1p2uBxzshnsAUFILnuFmE/HwFnnE/4orgj/j3nlkse+VPnPPw//jf+p2oWkIwJhD5u2loOvCcOr4E4kWkr59j6tkiE5EzH0Cu+5To9KO5M+Q5nim9miXP/4ErH5vPV1t3+ztCY0wn83UiUOADEVkqIte2sD4V2O71PMddZnyt95HIFW/AlW+RMGgsvw6ZzWMFV5D19E3c9NR7LN9eYjUEYwJEsI+PP1FVc0WkF/ChiKxX1YUHexA3iVwLMGDAgI6OMXCJwKAT8Aw6AfKWE77o/zFr3ZvU5bzLnH+cwEPxF3H8MccwfUw/esWE+ztaY4yPSGf96hORO4EKVX3Aa9k/gAWq+pL7fAMwRVXzWztOZmamZmV9p9/ZdJSizdQuepCglbORxjreaxjHU41nET90PBccncbJh/cmNNjfLYrGmIMlIktb6av1XdOQiESJSEzTPHAKsLrZZm8CV7hnDx0HlLaVBEwnSDqM0HMeIejnq/FMuoVTIzfwWsjvuWHbz5jz0lNMve8jXlz8LbX1dqtMY3oKn9UIRGQw8Lr7NBh4UVXvFZFZAKr6hIgI8ChwGlAJXKWqbf7ctxpBJ6sph2XPo1/+HSnNIS+oH09XT+Wz6FOZedIYzj86jZAgqyEY09W1VSPotKahjmKJwE8a6mDNXHTJU8j2L6khlLcajuPDyDM56aQzOHtMKuEhQf6O0hjTCksEpmPtWI0ueZqGFS8TXL+H1Y3pvOo5Bcm4gLPGDWN0/3icyp4xpquwRGB8o6YcXfkKlZ89SVTJevZoOK83TOCT2LMYc8wkzhuTRp84O9vImK7AEoHxLVXIyaJu8VPI2tcJbqzh68YhvNR4Eo2HT2fmlJGMTI3zd5TGBDRLBKbzVBXDitnULn6a0OKNVGoY8xqPZW2vaUw6eTpTRvS2ZiNj/MASgel8qrB9MbVL/wOrXyO0YQ/bG1P4OPwkBpz8E6YcY+MLGtOZLBEY/6qtpH7Nm+z+/FmSC76kXj1kDzyfYRfcBbH9/B2dMQHBLxeUGbNXaCTBYy6h10/fo/aG5Xwaewbp216l/sFR8N5voGKXvyM0JqBZIjCdKjw5nYk3PcddA5/jtdrxNH75BDw0CubfCaU5/g7PmIBkTUPGL+obGrnllRWsWrmUx9M+YHjBBwgKvUfC0FNg2KmQNg48dpGaMR3B+ghMl9TQqPz61ZXMWZrD7ceGcE2v9cg378O3X4A2QEQCDDkZhp/uPIbbKajGHKq2EoGvh6E2plVBHuG+848iLNjDHxd/y5upY5gy7FSmjA9ldO0ygjd/CBs/gFX/BU8IpE+EEWc6iSEuzd/hG9NjWI3A+J2q8uzn2byzMp+vt5fQ0KhEhwVz3OAkThiSwDHBmxlU9Alhm9+Dok3OTn0ynFrCYSdB/2MhONS/hTCmi7OmIdNtlFbV8cXmIhZuLGDhNwXkFFftXZcSE8akhGJOC17GqKrF9CpdgTTWQ2g0pE+CISc5U+JgP5bAmK7JEoHpllSVnOIqNu4qZ+POCjbuqmCTO1XU1BNNJWdGb+KcmHUcVbOUqEr3rKOkIU6H85CTYeAECLHxjoyxRGB6FFVla+EePttcxOebCvl8cxGlVbWkyw4ujv+G08JWMrB8GZ6GGgiJhEEnOFP/Y6HPUdaMZAKSJQLTozU0Kmvzyvh0UyH/W7+TrG3FhGkNZ8Vt5qLYdWRULSG84ltn46AwSB0L/Y9xEkPaOIju5d8CGNMJLBGYgFJYUcP8tTv5YO1OPt1YSG1DI309JZwev52J4Zs5on49vSrW4Wmsc3aIH+gkhP7HQFom9M6wWoPpcSwRmIBVUVPPom8KWJVbyoYd5azfUU5uSRVh1HKkZDMlcisTI7IZXreOqBp3qIugUOh1uHNmUp9R0Pco6H0khMX4tzDGfA+WCIzxUlpVx4Yd5azJK2XZtyUs21ZMbkkVfSji2JDNnBibQ0bQt6RWbySstnjfjr1HOlc8Dz8D+o0Fj43QYroPSwTGHEB+aRXLtpWwdFsxy74tZm1+GbX1DfSmmMzwHKbE5nEcq0krX4FoA0T1gmGnwLDTnY7o8Fh/F8GYNlkiMOYg1TU08s3OclbllLIyt5SVOSWsyy8nurGck4NXcF70ao6uW0p4QwUqHqRPhnOq6sDjYcDxEJXk7yIYsx9LBMZ0gPLqOrK2FfPl5iK+3FLEutwixso3nJuwlXMSsgnbsRTqq52Nk4dDv9FOP0Pvkc5pq5YcjB9ZIjDGB8qq63hv1Q7ufnstQR7hr+cdzslxebDtM9i+GHasgrLcfTvE9HM6oRPSIWGgc7ZS02NEAtgtPI0PWSIwxoeyC/dww0vLWJ1bxlUT0rnt9BGEBbvDZ+8pgp2rYMdqJzEUrIeSbc69nb1FJrk1h6YaxEinVmGnsZoOYonAGB+rqW/gz++u55nPshmZGsujM8aSnhzV+g7VpVC8zUkKxducBLFzNexat695yRMCycOcWkSvEdDrCGc+Pt3OWDIHzRKBMZ3k/TU7uHXOSqpqG4iPDCEkyENIkBAS5CE4yEN8RAgZaXGMSotnVP84UuMjEO8moYZ6Z4TVnV41iF1roeTbfduERsOA42DQZBg82bkAzhKDOQBLBMZ0otySKp75dCsVNfXUNSh1DY3UNzZSW68UVNSwLq+M2oZGAJKiQhnVP56jByYwcUgyI1PjCPK00FdQUw4FG5ykkLccti6Eoo3OuohEGDTJOVspaQgkDYa4ARBktxsx+/g1EYhIEJAF5KrqtGbrZgL3A009ao+q6lNtHc8SgenuausbWb+jjBU5pazYXsLy7SVs2lUBQHxkCMcflsSEIclMHJLMgMTI/WsM3sryYMsnsPUT57E8b986T7DTCZ10mNO81Hukc3V0ynAIDuuEUpquxt+J4BYgE4htJRFkquoN7T2eJQLTExWU1/D55kI+3VjIp5sKyS91+gnCgj30jQunT1w4/eIi6BsfTq+YcEKCPAR5IMjjPHqAZCnhiLBCEqq2w+4tsHszFG2Bwm+gocZ5IQmC5KFOUkga6ty7oWmKTLQzl3owv92qUkTSgDOBe4FbfPlaxnRnKTFhTB+dyvTRqagqWwr38MXmIr7dXUleSRX5pdUs3rqbHWXVNDS2/eOtV0xfMlJHMDI1joyRcQxMCCWxJoeY0g2EFq6DnWtg+xJY/RrgdaywOEgc5NQakodByghnPmGQNTP1cL7+6z4I3Aq0NVrX+SJyAvAN8HNV3d58AxG5FrgWYMCAAb6I05guQ0Q4LCWaw1Kiv7OuoVEprqylvkFpUKWh6bFRKayoYXVuKWvyyliVW8r/Nuxi/wp/BGHBmcRFjCchMpQxh4fzg341ZMaUEFed49QiijZB9mew8uV9uwWFOn0PvY90zlxqamaK7Wc1iB7CZ01DIjINOENVrxeRKcAvW2gaSgIqVLVGRH4CXKyqJ7Z1XGsaMqZ99tTUsza/jPzSakqr6iirqqO0qo7SyjoKK2rI2lZMaZUzFPeIPjFMHJLMlOG9mDAkCamtcJqUCr5xz1xa53RUl3r9TguPg15HQm/3tNZeRzqPEfF+KrFpi1/6CETkT8DlQD0QDsQCr6nqD1vZPgjYrapxbR3XEoExHaOhUVmTV8qnm5y+iazsYmobGpkwJIn/OyeDQS1dB1FV4iSFnaudJqamBFFTtm+b2FSnHyJpqPs4xHmMTbPTXP3I76ePtlEj6Kuq+e78ucCvVfW4to5licAY36iqbWDO0u3c994GahoauWHqEH4yefC+q6Rbo+oMpbFzrZMUdq2Fwo1OM5N3gggKg9i+TqKI7QcxXvNxqU6iiEqxZOEjfussbiWYu4EsVX0T+JmInI1Ta9gNzOzseIwxjojQIC4fn86pR/bhrrfX8rcPv2Hu8lz+eG4Gxw1uY8A8EYhLc6Zhp+xbrgoVu5zrHQo3OmcxleU7p73mLHEeG2r3P5YnxEkMsanOmUzJQ/d1Xiekg+cASckcErugzBjTogUbdvH7N1azfXcVk4YmkxofQVxkCPERocRHhpAQGUJ4SBDBHg9BHiE4SAjyCCEeD8kxofSKCW/54rgmqlBZ5NQmSnPdx5x9j0WbYc+ufdsHhULiYRDT2xmbqWmKSHTuO90nw0ke1oHdIr83DXUkSwTGdJ6q2gYe/XgjH67dSUllHSWVdXuvij6QII/QOyaMPnHh9I2PIC0+giG9ohnRJ5ahvaMJD2nHr/uqYqc2UbDB6bwu2gR7CpwEUrkbqkv23z48DvqOgr6jnWHA4wY4YzfV17iP7hQW6zZHpTo3GQqA5ihLBMaYDqGqVNU1UFJZR3FlLTX1jTQ0qnM6a6O6Q2k0UlBRQ35JNXmlVeSXVLOjrJrc4qq9ScQjkJ4UxbDeMQxMiiQ6LJiosOC9j1FhQfSKCad/YgQx4SGtB9RQ7ySLshzIX+EMv5G/3OnIbt7s1BpPiNN3EdffuY4iaYjT0Z00xHneQ67EtkRgjPG7+oZGtu2uZMOOctbvKOebHeVs2FlObkkVtfWt1zLiI0PonxBJ/8QI0hIi6RUTRnK0O8WEkhwdRkJk6P7NUPW1Tqd1xS4ICYdg7ynMqUm0pzlKPM5QHU0X1zU9Jg+DsO9e59GVWSIwxnRpdQ2N7Kmpp6Jpqq5nZ1kN24srySmuZPvuKne+5aQREiSkJ0UxpJdzId6QXvumdjVBeasudZqgijY7j03NUoUbobFu33ZNZz01nfEU554BFdXLudFQZKLz2EVqFJYIjDE9gqpSVl1PQXkNhRXuVF5Dfmk1mwv2sLmggm1Fe2gahSM2PJhLjx3IlccPpG9cxPd78YZ6KN7qXmC3HoqznSappppFXWXL+4VEOUkhuhdE94EY76mfczZU/ACf34TIEoExJmDU1DeQXVjJpl0VzFuVz7ur8/GIMO2ovvx40mBGprZ5zeqhUXX7KnL3dWRXFe+bKougYieU74TyfKja3ewA7im4CelOv0Rcf+eMqKgUiEre9xgef8hnRVkiMMYErO27K3n282xeXrKdipp6jhucSEZqHDX1jdTUNVJd30BNnXPPiITIUPrEhdM7Npw+sc6or71iwogJDyE8xNP6kOAHq77GSQylOU7Nojgbdm9157c6Z0a1ZPwNcOq9h/SSlgiMMQGvrLqOV5Zs599fZFNUUUtYsIew4CDCQjyEBXsI8ngocpubWhrgNdgjRIc7ZzY1TZFhwUSFBhEZ6pzpFBkazIDESEamxjKsd8zB9080qa91ahF7CqCyEPYUOvN9R0H6xEM6pCUCY4xpp/qGRgoratlRVs2O0moKKmqoqK6noqaO8mqnI7vc7dCurGugsqaeytoG9tTWs8e9Kx04iWNIr2hGpsYxvHcMdY2NlFXV7x0AsKy6jrqGRgYkRjIoOZpByVEMSo5iYFLkoSeQNnSpISaMMaYrCw7y0Me9GRD9D25fVWX77ipW55XuHRJ8wYZdzFmaA0BokIfYiBDiIoKJjQjBI8L/1hdQWJGz9xgiEBseQoh7pXawx0NwkBDsEWYcM4AfTxrckcUFLBEYY0yHEREGJEUyICmSMzL6Ak5yKK2qIyw4qNV+hvLqOrILK9lSWMHWwj0U76ml3r1Qr969UK++UUmO9s2pqJYIjDHGh0SE+Mi2Tw2NCQ8hIy2OjDQfnNHUDj1/gA1jjDFtskRgjDEBzhKBMcYEOEsExhgT4CwRGGNMgLNEYIwxAc4SgTHGBDhLBMYYE+C63VhDIlIAbDvI3ZKBQh+E05VYGXuOQChnIJQRulY5B6pqSksrul0iOBQiktXaYEs9hZWx5wiEcgZCGaH7lNOahowxJsBZIjDGmAAXKIngSX8H0AmsjD1HIJQzEMoI3aScAdFHYIwxpnWBUiMwxhjTCksExhgT4Hp0IhCR00Rkg4hsEpHb/B3PgYjIv0Rkl4is9lqWKCIfishG9zHBXS4i8rBbtpUiMtZrnyvd7TeKyJVey48WkVXuPg9LS7dK8jER6S8iH4vIWhFZIyI39dByhovIVyKywi3nXe7yQSKy2I3tZREJdZeHuc83uevTvY71G3f5BhE51Wt5l/h8i0iQiHwtIm+7z3tiGbPdz9RyEclyl/Wcz6yq9sgJCAI2A4OBUGAFcIS/4zpAzCcAY4HVXsvuA25z528D/uLOnwG8CwhwHLDYXZ4IbHEfE9z5BHfdV+624u57uh/K2BcY687HAN8AR/TAcgoQ7c6HAIvdmF4BLnGXPwFc585fDzzhzl8CvOzOH+F+dsOAQe5nOqgrfb6BW4AXgbfd5z2xjNlAcrNlPeYz25NrBMcAm1R1i6rWArOB6X6OqU2quhDY3WzxdODf7vy/gXO8lj+nji+BeBHpC5wKfKiqu1W1GPgQOM1dF6uqX6rzyXvO61idRlXzVXWZO18OrANS6XnlVFWtcJ+GuJMCJwJz3OXNy9lU/jnASe6vwunAbFWtUdWtwCacz3aX+HyLSBpwJvCU+1zoYWVsQ4/5zPbkRJAKbPd6nuMu6256q2q+O78D6O3Ot1a+tpbntLDcb9ymgTE4v5Z7XDndJpPlwC6cf/rNQImq1rcQ297yuOtLgSQOvvyd7UHgVqDRfZ5EzysjOEn8AxFZKiLXust6zGfWbl7fjaiqikiPON9XRKKBV4GbVbXMu0m0p5RTVRuA0SISD7wOjPBzSB1KRKYBu1R1qYhM8Xc8PjZRVXNFpBfwoYis917Z3T+zPblGkAv093qe5i7rbna6VUfcx13u8tbK19bytBaWdzoRCcFJAi+o6mvu4h5XziaqWgJ8DIzHaSZo+gHmHdve8rjr44AiDr78nWkCcLaIZOM025wIPETPKiMAqprrPu7CSerH0JM+s53ZIdGZE05tZwtO51NTR9OR/o6rHXGns39n8f3s3yF1nzt/Jvt3SH3lLk8EtuJ0RiW484nuuuYdUmf4oXyC0wb6YLPlPa2cKUC8Ox8BLAKmAf9l/47U6935n7J/R+or7vyR7N+RugWnE7VLfb6BKezrLO5RZQSigBiv+c+B03rSZ9YvH5pO/AOegXNWymbgt/6Opx3xvgTkA3U47YQ/wmlD/QjYCMz3+uAI8He3bKuATK/jXI3T4bYJuMpreSaw2t3nUdwryzu5jBNx2ltXAsvd6YweWM6jgK/dcq4G/uAuH+z+029yvzDD3OXh7vNN7vrBXsf6rVuWDXidTdKVPt/snwh6VBnd8qxwpzVNcfSkz6wNMWGMMQGuJ/cRGGOMaQdLBMYYE+AsERhjTICzRGCMMQHOEoExxgQ4SwQm4IhIhfuYLiKXdvCxb2/2/POOPL4xvmCJwASydOCgEoHXFbOt2S8RqOrxBxmTMZ3OEoEJZH8GJrljzP/cHSTufhFZ4o4j/xMAEZkiIotE5E1grbtsrjsA2ZqmQchE5M9AhHu8F9xlTbUPcY+92h13/mKvYy8QkTkisl5EXmgai15E/izOfRtWisgDnf7umIBhg86ZQHYb8EtVnQbgfqGXquo4EQkDPhORD9xtxwIj1RkmGeBqVd0tIhHAEhF5VVVvE5EbVHV0C691HjAaGAUku/ssdNeNwRlmIQ/4DJggIuuAc4ERqqruwHXG+ITVCIzZ5xTgCnfo6MU4QwgMddd95ZUEAH4mIiuAL3EGEhtK2yYCL6lqg6ruBD4BxnkdO0dVG3GG3EjHGaK5GnhaRM4DKr936YxphSUCY/YR4EZVHe1Og1S1qUawZ+9GzpDLJwPjVXUUzphC4d/jdWu85huAYHXG6z8G5wYu04D3vsfxjWmTJQITyMpxbpfZ5H3gOneYbERkmIhEtbBfHFCsqpUiMgJn1MgmdU37N7MIuNjth0jBuS3pV60F5t6vIU5V5wE/x2lSMsYnrI/ABLKVQIPbxPMszlj66cAyt8O2gJZvGfgeMMttx9+A0zzU5ElgpYgsU9XLvJa/jnM/ghU4o6/eqqo73ETSkhjgDREJx6mp3HJoRTTmwGz0UWOMCXDWNGSMMQHOEoExxgQ4SwTGGBPgLBEYY0yAs0RgjDEBzhKBMcYEOEsExhgT4P4/46+gkhWkpu8AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wjsy6zxNL2lV"
      },
      "source": [
        "Let’s quickly save our trained model:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VAVk7xe-L2lV"
      },
      "source": [
        "PATH = '/content/drive/My Drive/Transformer2.pth'\n",
        "torch.save(model2.state_dict(), PATH)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YcpvhUscL2lX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "56f3f0a6-e7cc-4f6e-c99d-c31d45fe3270"
      },
      "source": [
        "print(train_losses2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[7.044583921909332, 6.3595043592453, 6.07445554971695, 5.8681496868133545, 5.715434666633606, 5.575719193935394, 5.4684157176017765, 5.378194270133972, 5.309905904769898, 5.174778644561767, 5.119003778934479, 5.070746387004852, 5.030598823547363, 4.988202484130859, 4.95817217206955, 4.9176596202850344, 4.887787534236908, 4.851358037948608, 4.773772130012512, 4.751644444465637, 4.73858655834198, 4.717497355937958, 4.692257964611054, 4.677892746448517, 4.65961946439743, 4.643309813976288, 4.623094560623169, 4.5636784477233885, 4.5618979017734524, 4.5451003775596615, 4.523619549274445, 4.51459657406807, 4.513636318206787, 4.497703789234161, 4.492364508628845, 4.4708758561611175, 4.420891220331192, 4.415192759990692, 4.413903271436691, 4.404896262645721, 4.390912525653839, 4.386796081542969, 4.376458330154419, 4.369233320474625, 4.356462274551392, 4.320657784461975, 4.32321314573288, 4.314762910604477, 4.310287416934967, 4.30654602098465, 4.291003308773041, 4.296266507863998, 4.284498723983765, 4.283087710857392]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8hHgYOV9L2lX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "143df632-f022-4fb5-9315-8d1178439f5c"
      },
      "source": [
        "print(validation_losses2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[6.551886562984976, 6.183807052929306, 5.9499443915266506, 5.776914807149565, 5.632861478433333, 5.5206726271899145, 5.420868626940929, 5.337267530621156, 5.265813850666406, 5.165035012723351, 5.115109113752007, 5.063746893550613, 5.017508433757821, 4.9798573432516315, 4.943034940073137, 4.9101179707951905, 4.8776909127400145, 4.847597273569258, 4.806284184104143, 4.782784612906747, 4.758095909122214, 4.735826215378933, 4.712764090850573, 4.694577262069887, 4.676892681727334, 4.661713894007921, 4.637559575081762, 4.618094678018608, 4.603134909041233, 4.592768702520376, 4.578026972564996, 4.564280779978363, 4.543269932548316, 4.531779039028534, 4.517388723143676, 4.5061694053040835, 4.493172692762815, 4.478880666328523, 4.4649485103667725, 4.452687330495298, 4.444065264960964, 4.4349572400307675, 4.422237808519608, 4.416032886416081, 4.399968991426527, 4.402589618547513, 4.3935330673203525, 4.381490412657779, 4.368314714547521, 4.359261930378434, 4.353516557971303, 4.345493801279006, 4.334650267183391, 4.327860888758071]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0kIf7jceUmQP"
      },
      "source": [
        "## Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 267,
          "referenced_widgets": [
            "3de110efdac44dcf9f1e31ed65ac8d1f",
            "1715549bfc5442ba9625d09fae2f43f9",
            "cae998265daf4020a4750099034ee728",
            "54547d707e5e464a9c7fbe1bf0a2f4cb",
            "cbc761ef25e548a59ee71d05d3f256b5",
            "340bb5233aed4461a351a2743c044041",
            "201613042cc7418e80f9be0e644613df",
            "e9701d28ec8d475d8fcdd90ffaf2b20f"
          ]
        },
        "id": "LthNG7cYUk6V",
        "outputId": "1a11a9c1-c3f0-47b0-81ba-d346166bd0e8"
      },
      "source": [
        "torch.manual_seed(2045)\n",
        "\n",
        "pad_idx = 0\n",
        "\n",
        "# Parameters\n",
        "params = {'batch_size': 1,\n",
        "          'shuffle': False,\n",
        "          'num_workers':2,\n",
        "          'collate_fn' : BatchPadCollate(pad_idx=pad_idx)\n",
        "          }\n",
        "\n",
        "# Training and validation Data Generators\n",
        "test_set0 = AFEC_Dataset(en_test_dic_,fa_test_dic_0,partition2['test'],en_vocab_dic_, fa_vocab_dic_)\n",
        "test_generator0 = torch.utils.data.DataLoader(test_set0, **params)\n",
        "test_set1 = AFEC_Dataset(en_test_dic_,fa_test_dic_0,partition2['test'],en_vocab_dic_, fa_vocab_dic_)\n",
        "test_generator1 = torch.utils.data.DataLoader(test_set1, **params)\n",
        "test_set2 = AFEC_Dataset(en_test_dic_,fa_test_dic_0,partition2['test'],en_vocab_dic_, fa_vocab_dic_)\n",
        "test_generator2 = torch.utils.data.DataLoader(test_set2, **params)\n",
        "test_set3 = AFEC_Dataset(en_test_dic_,fa_test_dic_0,partition2['test'],en_vocab_dic_, fa_vocab_dic_)\n",
        "test_generator3 = torch.utils.data.DataLoader(test_set3, **params)\n",
        "\n",
        "\n",
        "# Hyperparameters\n",
        "embed_dim = 256\n",
        "num_heads = 8\n",
        "dim_key = 64\n",
        "dim_value = 64\n",
        "dim_inner = 1024\n",
        "num_layers = 3\n",
        "src_vocab_size = len(en_vocab_dic_.get_str_2_idx())\n",
        "trg_vocab_size = len(fa_vocab_dic_.get_str_2_idx())\n",
        "learning_rate = 1e-4\n",
        "\n",
        "kwargs = {'p_dropout': 0.1,\n",
        "          'kdim': dim_key,\n",
        "          'vdim':dim_value,\n",
        "          }\n",
        "# Model\n",
        "PATH = '/content/drive/My Drive/Transformer2.pth'\n",
        "model2 = Transformer(num_layers, embed_dim, dim_inner, num_heads, src_vocab_size,trg_vocab_size, device,**kwargs).to(device)\n",
        "model2.load_state_dict(torch.load(PATH))\n",
        "\n",
        "test_dataiter0 = iter(test_generator0)\n",
        "test_dataiter1 = iter(test_generator1)\n",
        "test_dataiter2 = iter(test_generator2)\n",
        "test_dataiter3 = iter(test_generator3)\n",
        "\n",
        "list_of_references = []\n",
        "hypothesises = []\n",
        "bleu_scores = []\n",
        "nist_scores = []\n",
        "\n",
        "# Loop over batches\n",
        "for num_batch in tqdm(range(len(test_generator0))):\n",
        "    # Test \n",
        "    with torch.set_grad_enabled(False):\n",
        "        # Transfer to GPU\n",
        "        batch_src0, batch_trg0 = test_dataiter0.next() \n",
        "        batch_src0, batch_trg0 = batch_src0.to(device), batch_trg0.to(device)\n",
        "        batch_src1, batch_trg1 = test_dataiter1.next() \n",
        "        batch_src1, batch_trg1 = batch_src1.to(device), batch_trg1.to(device)\n",
        "        batch_src2, batch_trg2 = test_dataiter2.next() \n",
        "        batch_src2, batch_trg2 = batch_src2.to(device), batch_trg2.to(device)\n",
        "        batch_src3, batch_trg3 = test_dataiter3.next() \n",
        "        batch_src3, batch_trg3 = batch_src3.to(device), batch_trg3.to(device)\n",
        "\n",
        "        # Model computations\n",
        "        # forward + backward + optimize\n",
        "        batch_outputs1 = model2(batch_src0, batch_trg0[: , :-1])\n",
        "        batch_outputs = model2.translate_sentence(batch_src0, batch_trg0.shape[1] + 5, 1, 2)\n",
        "        batch_outputs = torch.Tensor(batch_outputs)\n",
        "\n",
        "        references = [fa_vocab_dic_.IndexDecoding(batch_trg0[0,:]),fa_vocab_dic_.IndexDecoding(batch_trg1[0,:]),\n",
        "                      fa_vocab_dic_.IndexDecoding(batch_trg2[0,:]),fa_vocab_dic_.IndexDecoding(batch_trg3[0,:])]\n",
        "        list_of_references.append(references)\n",
        "        \n",
        "        hypothesises.append(fa_vocab_dic_.IndexDecoding(batch_outputs))\n",
        "        bleu_scores.append(sentence_bleu(references,fa_vocab_dic_.IndexDecoding(batch_outputs)))\n",
        "        nist_scores.append(sentence_nist(references,fa_vocab_dic_.IndexDecoding(batch_outputs)))\n",
        "        \n",
        "        # print(batch_src0.shape, batch_trg0.shape, len(batch_outputs))\n",
        "        # print(en_vocab_dic_.IndexDecoding(batch_src0[0,:])[1:-1])\n",
        "        # print(fa_vocab_dic_.IndexDecoding(batch_trg0[0,:])[1:-1])\n",
        "        # print(fa_vocab_dic_.IndexDecoding(batch_outputs)[1:-1])\n",
        "        # if num_batch == 5:\n",
        "        #   break\n",
        "\n",
        "bleu_score = corpus_bleu(list_of_references, hypothesises) \n",
        "nist_score = corpus_nist(list_of_references, hypothesises, n = 5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3de110efdac44dcf9f1e31ed65ac8d1f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=250.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/nltk/translate/bleu_score.py:490: UserWarning: \n",
            "Corpus/Sentence contains 0 counts of 3-gram overlaps.\n",
            "BLEU scores might be undesirable; use SmoothingFunction().\n",
            "  warnings.warn(_msg)\n",
            "/usr/local/lib/python3.7/dist-packages/nltk/translate/bleu_score.py:490: UserWarning: \n",
            "Corpus/Sentence contains 0 counts of 2-gram overlaps.\n",
            "BLEU scores might be undesirable; use SmoothingFunction().\n",
            "  warnings.warn(_msg)\n",
            "/usr/local/lib/python3.7/dist-packages/nltk/translate/bleu_score.py:490: UserWarning: \n",
            "Corpus/Sentence contains 0 counts of 4-gram overlaps.\n",
            "BLEU scores might be undesirable; use SmoothingFunction().\n",
            "  warnings.warn(_msg)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1b-y5YPgLfvC",
        "outputId": "f5a8d235-5f2a-4e62-cb93-ca0f7ad1b3de"
      },
      "source": [
        "print(bleu_score,nist_score)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.06977961030647861 0.0014485707827332276\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UT9mZ5BWU-t3"
      },
      "source": [
        "print(fa_vocab_dic_.IndexDecoding(torch.max(batch_outputs1, dim = 2)[1][0,:]))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 906,
          "referenced_widgets": [
            "f911da78d934418d95925f3ef31aaa76",
            "8e392dd6e4e6468490793e67cba56a92",
            "bdcb70d08770451ab76d0a5da13a0d7d",
            "75556d32df6441efbb2e0c9e39470cf0",
            "5fbeda627c6240f4a3ba266145b5436e",
            "b8fbbbc3246c4b04ab2a957e2dbe7011",
            "129d6e65690e475cb691403cdb9cbbc0",
            "204180c761ba4527aa12da24a8f95279"
          ]
        },
        "id": "zapex2N2MmfY",
        "outputId": "b476946b-a993-4989-c548-5415c9add343"
      },
      "source": [
        "torch.manual_seed(2045)\n",
        "\n",
        "pad_idx = 0\n",
        "\n",
        "# Parameters\n",
        "params = {'batch_size': 1,\n",
        "          'shuffle': True,\n",
        "          'num_workers':2,\n",
        "          'collate_fn' : BatchPadCollate(pad_idx=pad_idx)\n",
        "          }\n",
        "\n",
        "# Training and validation Data Generators\n",
        "test_set2 = AFEC_Dataset(en_test_dic_,fa_test_dic_0,partition2['test'],en_vocab_dic_, fa_vocab_dic_)\n",
        "test_generator2 = torch.utils.data.DataLoader(test_set2, **params)\n",
        "\n",
        "# Hyperparameters\n",
        "embed_dim = 256\n",
        "num_heads = 8\n",
        "dim_key = 64\n",
        "dim_value = 64\n",
        "dim_inner = 1024\n",
        "num_layers = 3\n",
        "src_vocab_size = len(en_vocab_dic_.get_str_2_idx())\n",
        "trg_vocab_size = len(fa_vocab_dic_.get_str_2_idx())\n",
        "learning_rate = 1e-4\n",
        "num_iterations = 10\n",
        "\n",
        "kwargs = {'p_dropout': 0.1,\n",
        "          'kdim': dim_key,\n",
        "          'vdim':dim_value,\n",
        "          }\n",
        "# Model\n",
        "# PATH = '/content/drive/My Drive/Transformer2.pth'\n",
        "# model2 = Transformer(num_layers, embed_dim, dim_inner, num_heads, src_vocab_size,trg_vocab_size, device,**kwargs).to(device)\n",
        "# model2.load_state_dict(torch.load(PATH))\n",
        "\n",
        "test_dataiter2 = iter(test_generator2)\n",
        "\n",
        "# Loop over epochs\n",
        "for itr in tqdm(range(num_iterations)):\n",
        "    \n",
        "    # Test\n",
        "    batch_src, batch_trg = test_dataiter2.next() \n",
        "    \n",
        "    with torch.set_grad_enabled(False):\n",
        "\n",
        "\n",
        "        # Model computations\n",
        "        # forward + backward + optimize\n",
        "        \n",
        "        batch_outputs = model2.translate_sentence(batch_src, batch_trg.shape[1] + 5, 1, 2)\n",
        "        batch_outputs = torch.Tensor(batch_outputs)\n",
        "\n",
        "        print(\"Sentence {}:\".format(itr + 1))\n",
        "        print(\" \".join(en_vocab_dic_.IndexDecoding(batch_src[0,:])).replace(\"<sos>\", \"\").replace(\"<eos>\", \".\"))\n",
        "        print(\" \".join(fa_vocab_dic_.IndexDecoding(batch_trg[0,:])).replace(\"<sos>\", \"\").replace(\"<eos>\", \"\").replace(\"<unk>\",\"ن.ش\"))\n",
        "        print(\" \".join(fa_vocab_dic_.IndexDecoding(batch_outputs)).replace(\"<sos>\", \"\").replace(\"<eos>\", \"\").replace(\"<unk>\",\"ن.ش\"))\n",
        "        print(\"-\"*50)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f911da78d934418d95925f3ef31aaa76",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Sentence 1:\n",
            " once again shall we stay at the hotel grun￭ sch￭ nab￭ el .\n",
            " یکبار دیگر آیا قرار است در هتل گران￭ ش￭ ابل بمانیم \n",
            " زمانی که ما در هتل هتل های هتل را در هتل ببینیم \n",
            "--------------------------------------------------\n",
            "Sentence 2:\n",
            " fine let us go for a little walk in hamburg .\n",
            " خوب است اجازه دهید برای یک کوتاه در هامبورگ برویم \n",
            " خوب است اجازه دهید برای ما در یک مکان کوچک \n",
            "--------------------------------------------------\n",
            "Sentence 3:\n",
            " yes i have already booked the hotel .\n",
            " بله من در حال حاضر هتل را رزرو \n",
            " بله من از هتل رزرو کردم \n",
            "--------------------------------------------------\n",
            "Sentence 4:\n",
            " in the evening we should watch the newest movie with arnold schwarzenegger .\n",
            " عصر باید جدیدترین فیلم آرنولد شوا￭ ن.ش را تماشا کنیم \n",
            " ما باید با فیلم فیلم فیلم ساز در عصر را با یک فیلم در سال به نمایش\n",
            "--------------------------------------------------\n",
            "Sentence 5:\n",
            " where should we meet .\n",
            " کجا باید ملاقات کنیم \n",
            " ما باید به این ترتیب دهیم که به توافق رسید \n",
            "--------------------------------------------------\n",
            "Sentence 6:\n",
            " hello i would prefer the intercontinental .\n",
            " سلام من این￭ ترک￭ انتی￭ ننتال را ترجیح میدادم \n",
            " سلام من ترجیح میدهم \n",
            "--------------------------------------------------\n",
            "Sentence 7:\n",
            " i have asked when we will arrive in hanover .\n",
            " من چه وقت به هانوور خواهیم رسید \n",
            " وقتی که در سال به ما از ما درخواست کردیم \n",
            "--------------------------------------------------\n",
            "Sentence 8:\n",
            " fine let us meet at the station at nine oc￭ lock on thursday morning .\n",
            " خوب است اجازه دهید ساعت نه صبح ن.ش در ایستگاه ملاقات کنیم \n",
            " خوب بگذارید صبح ساعت نه صبح همدیگر را ببینیم \n",
            "--------------------------------------------------\n",
            "Sentence 9:\n",
            " yes i have got a timetable when would you like to go .\n",
            " بله من یک جدول زمانی دارم چه وقت دوست داری بروی \n",
            " بله وقتی من یک بار دیگر وقتی که یک لحظه به شما گفتم \n",
            "--------------------------------------------------\n",
            "Sentence 10:\n",
            " i agree everything okay .\n",
            " موافقم همه چیز مرتب است \n",
            " باشه من به توافق همه چیز را موافقت میکنم \n",
            "--------------------------------------------------\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dAKCHW0DSEx3"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_xnKPyuTDgyB"
      },
      "source": [
        "# Part 3:"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FQNQ-1XjDgyB"
      },
      "source": [
        "## Preprocessing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yDQEaYFMDgyC"
      },
      "source": [
        "### Load the dataset to colab directory "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cKJoBf_jDgyC"
      },
      "source": [
        "For the sake of speed, copy the dataset files to the root directory of notebook and unziped them. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qy4_ZSsKDgyC"
      },
      "source": [
        "!cp /content/drive/My\\ Drive/AFEC-merged-all.zip /content/AFEC-merged-all.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zm-eRA-mDgyD"
      },
      "source": [
        "!cp /content/drive/My\\ Drive/Test.zip /content/Test.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Rsx90kOGDgyD",
        "outputId": "bb3a06b9-3dae-4323-b306-9e644e76fd2b"
      },
      "source": [
        "!unzip /content/AFEC-merged-all.zip"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/AFEC-merged-all.zip\n",
            "  inflating: AFEC-merged.en          \n",
            "  inflating: AFEC-merged.fa          \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z8lIBNrUDgyE"
      },
      "source": [
        "os.rename('/content/AFEC-merged.en',\"/content/AFEC-merged_en.txt\")\n",
        "os.rename('/content/AFEC-merged.fa',\"/content/AFEC-merged_fa.txt\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dpRuSES6DgyF",
        "outputId": "8a03c946-3fa4-4811-f2a4-cd74b8b40726"
      },
      "source": [
        "!unzip /content/Test.zip"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/Test.zip\n",
            "   creating: Test/\n",
            "  inflating: Test/test.en            \n",
            "  inflating: Test/test.fa0           \n",
            "  inflating: Test/test.fa1           \n",
            "  inflating: Test/test.fa2           \n",
            "  inflating: Test/test.fa3           \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DJVkuGWVDgyF"
      },
      "source": [
        "os.rename('/content/Test/test.en',\"/content/test_en.txt\")\n",
        "os.rename('/content/Test/test.fa0',\"/content/test_fa0.txt\")\n",
        "os.rename('/content/Test/test.fa1',\"/content/test_fa1.txt\")\n",
        "os.rename('/content/Test/test.fa2',\"/content/test_fa2.txt\")\n",
        "os.rename('/content/Test/test.fa3',\"/content/test_fa3.txt\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "16I_ajrjE33T"
      },
      "source": [
        "### Vocabulary "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sZi_RAASE33U"
      },
      "source": [
        "class Vocabulary_and_Dictionary():\n",
        "\n",
        "    def __init__(self, freq_threshold = 2):\n",
        "        self.freq_threshold = freq_threshold\n",
        "        \n",
        "        self.frequencies = {}\n",
        "        self.idx_2_str = {0: \"<pad>\", 1: \"<sos>\", 2: \"<eos>\", 3: \"<unk>\"}\n",
        "        self.str_2_idx = {\"<pad>\": 0, \"<sos>\": 1, \"<eos>\": 2, \"<unk>\": 3}\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.idx_2_str)\n",
        "\n",
        "    @staticmethod\n",
        "    def preprocess(text):\n",
        "        # prepare translation table for removing punctuation\n",
        "        table = str.maketrans('', '', string.punctuation)\n",
        "\n",
        "        # tokenize on white space\n",
        "        text = text.split()\n",
        "        # convert to lower case\n",
        "        text = [word.lower() for word in text]\n",
        "        # remove punctuation from each token\n",
        "        text = [word.translate(table) for word in text]\n",
        "\n",
        "        # remove tokens with numbers in them\n",
        "        text = [word for word in text if word.isalpha()]\n",
        "        \n",
        "        return text\n",
        "\n",
        "    def build_vocabulary_and_dictionary(self, text_file_root):\n",
        "\n",
        "        self.dic = {}\n",
        "        \n",
        "        idx = 4\n",
        "        lines = [line.strip() for line in open(text_file_root)]\n",
        "        \n",
        "        for i in range(len(lines)):\n",
        "\n",
        "            words =  self.preprocess(lines[i])\n",
        "            for word in words:\n",
        "                if word not in self.frequencies:\n",
        "                    self.frequencies[word] = 1\n",
        "\n",
        "                else:\n",
        "                    self.frequencies[word] += 1\n",
        "\n",
        "                if self.frequencies[word] == self.freq_threshold:\n",
        "                    self.str_2_idx[word] = idx\n",
        "                    self.idx_2_str[idx] = word\n",
        "                    idx += 1\n",
        "\n",
        "            self.dic[i] = [\" \".join(words)] \n",
        "\n",
        "    def IndexEncoding(self, text):\n",
        "        tokenized_text = self.preprocess(text)\n",
        "        index_encoded_text = [self.str_2_idx[token] if token in self.str_2_idx else self.str_2_idx[\"<unk>\"]\n",
        "                                for token in tokenized_text]\n",
        "        \n",
        "        return index_encoded_text\n",
        "\n",
        "    def IndexDecoding(self, sen_tensor):\n",
        "        tokenized_vec = list(sen_tensor.cpu().numpy())\n",
        "        decoded_vec = [self.idx_2_str[token] for token in tokenized_vec]\n",
        "        return decoded_vec\n",
        "\n",
        "    def get_str_2_idx(self):\n",
        "        return self.str_2_idx\n",
        "\n",
        "    def get_dic(self):\n",
        "        return self.dic"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d8DvoxIDE33V",
        "outputId": "3f42f0e7-d16e-4663-8d00-ec5e9d13a414"
      },
      "source": [
        "en_vocab_dic = Vocabulary_and_Dictionary(freq_threshold = 10)\n",
        "\n",
        "train_text_root = \"/content/AFEC-merged_en.txt\"\n",
        "en_vocab_dic.build_vocabulary_and_dictionary(train_text_root)\n",
        "\n",
        "en_train_dic = en_vocab_dic.get_dic() \n",
        "\n",
        "print(len(en_vocab_dic.get_str_2_idx()))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "30032\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NhlMngcWE33W",
        "outputId": "2889d0fc-db6b-42e1-d0b8-d7a2a718f624"
      },
      "source": [
        "en_train_dic[0],len(en_train_dic)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['north waziristan operation kills more militants'], 684417)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p5wxICdqE33X"
      },
      "source": [
        "fa_vocab_dic = Vocabulary_and_Dictionary(freq_threshold = 10)\n",
        "\n",
        "train_text_root1 = \"/content/AFEC-merged_fa.txt\"\n",
        "fa_vocab_dic.build_vocabulary_and_dictionary(train_text_root1)\n",
        "\n",
        "fa_train_dic = fa_vocab_dic.get_dic() \n",
        "\n",
        "print(len(fa_vocab_dic.get_str_2_idx()))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sIHJ9zS3E33Y"
      },
      "source": [
        "fa_train_dic[0],len(fa_train_dic)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4tpkZLXjE33Y"
      },
      "source": [
        "en_vocab_dic2 = Vocabulary_and_Dictionary(freq_threshold = 2)\n",
        "\n",
        "test_text_root = \"/content/test_en.txt\"\n",
        "en_vocab_dic2.build_vocabulary_and_dictionary(test_text_root)\n",
        "\n",
        "en_test_dic = en_vocab_dic2.get_dic() \n",
        "\n",
        "print(len(en_vocab_dic2.get_str_2_idx()))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pdEcGnWeE33Z"
      },
      "source": [
        "fa_vocab_dic2 = Vocabulary_and_Dictionary(freq_threshold = 2)\n",
        "\n",
        "test_text_root0 = \"/content/test_fa0.txt\"\n",
        "fa_vocab_dic2.build_vocabulary_and_dictionary(test_text_root0)\n",
        "\n",
        "fa_test_dic0 = fa_vocab_dic2.get_dic() \n",
        "\n",
        "test_text_root1 = \"/content/test_fa1.txt\"\n",
        "fa_vocab_dic2.build_vocabulary_and_dictionary(test_text_root1)\n",
        "\n",
        "fa_test_dic1 = fa_vocab_dic2.get_dic() \n",
        "\n",
        "test_text_root2 = \"/content/test_fa2.txt\"\n",
        "fa_vocab_dic2.build_vocabulary_and_dictionary(test_text_root2)\n",
        "\n",
        "fa_test_dic2 = fa_vocab_dic2.get_dic() \n",
        "\n",
        "test_text_root3 = \"/content/test_fa3.txt\"\n",
        "fa_vocab_dic2.build_vocabulary_and_dictionary(test_text_root3)\n",
        "\n",
        "fa_test_dic3 = fa_vocab_dic2.get_dic() \n",
        "\n",
        "print(len(fa_vocab_dic2.get_str_2_idx()))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TyJkbcoIE33a"
      },
      "source": [
        "en_test_dic"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JKf_tNYDE33b"
      },
      "source": [
        "fa_vocab_dic2.get_str_2_idx()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5mSjcNfwDgyJ"
      },
      "source": [
        "### AFEC Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7WRo0xPqDgyK"
      },
      "source": [
        "class AFEC_Dataset(torch.utils.data.Dataset):\n",
        "    \"\"\"\n",
        "    Characterize AFEC Dataset for PyTorch. To be compatible with \n",
        "    DataLoader and other PyTorch syntaxes.\n",
        "    \"\"\"\n",
        "    def __init__(self, src_dic, trg_dic, indices, src_vocab, trg_vocab):\n",
        "        \n",
        "        self.src_dic = src_dic\n",
        "        self.trg_dic = trg_dic\n",
        "\n",
        "        # Get img names, caption columns\n",
        "        self.src_sentences = [self.src_dic[i] for i in indices]\n",
        "        self.trg_sentences = [self.trg_dic[i] for i in indices]\n",
        "\n",
        "        self.src_vocab = src_vocab\n",
        "        self.trg_vocab = trg_vocab \n",
        "            \n",
        "    def __len__(self):\n",
        "        'Denotes the total number of samples'\n",
        "        return len(self.src_sentences)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        \"\"\" \n",
        "        Generates one sample of data with label\n",
        "        \n",
        "        Arguments\n",
        "        ------------\n",
        "        index : integer, in range of len(self.list_IDs)\n",
        "                index of sample in list of IDs,\n",
        "        \n",
        "        Returns\n",
        "        ------------\n",
        "        X : torch tensor,\n",
        "            input data\n",
        "\n",
        "        Y : torch tensor,\n",
        "            correspond label of input data\n",
        "        \"\"\"\n",
        "        # Select sample\n",
        "        src_sentence = [ self.src_vocab.str_2_idx[\"<sos>\"] ]\n",
        "        src_sentence += self.src_vocab.IndexEncoding(self.src_sentences[index][0])\n",
        "        src_sentence.append(self.src_vocab.str_2_idx[\"<eos>\"])\n",
        "\n",
        "        trg_sentence = [ self.trg_vocab.str_2_idx[\"<sos>\"] ]\n",
        "        trg_sentence += self.trg_vocab.IndexEncoding(self.trg_sentences[index][0])\n",
        "        trg_sentence.append(self.trg_vocab.str_2_idx[\"<eos>\"])\n",
        "       \n",
        "        tensor_src_sentence = torch.tensor(src_sentence)\n",
        "        tensor_trg_sentence = torch.tensor(trg_sentence)\n",
        "\n",
        "        return tensor_src_sentence , tensor_trg_sentence"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yTuQEC9XDgyK"
      },
      "source": [
        "### Split Dataset and Activate CUDA"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AiFY6YL6DgyK"
      },
      "source": [
        "#train/val/test split \n",
        "random.seed(2045)\n",
        "\n",
        "indices = list(range(len(en_train_dic)))\n",
        "num_samples = len(en_train_dic)\n",
        "random.shuffle(indices)\n",
        "\n",
        "train_indices = indices[0 : int(0.9 * num_samples)]\n",
        "validation_indices = indices[int(0.9 * num_samples) : ]\n",
        "test_indices = list(range(len(en_test_dic)))\n",
        "\n",
        "partition = {'train' : train_indices , 'validation' : validation_indices,\n",
        "             'test' : test_indices }"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PBbjqHTCDgyL"
      },
      "source": [
        "# CUDA for PyTorch\n",
        "use_cuda = torch.cuda.is_available()\n",
        "device = torch.device(\"cuda:0\" if use_cuda else \"cpu\")\n",
        "torch.backends.cudnn.benchmark = True"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A5xGp1nZDgyL"
      },
      "source": [
        "device"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PwGhTbhnDgyL"
      },
      "source": [
        "len(partition['test'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kcnVOV7ADgyM"
      },
      "source": [
        "### Padding"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XAYue3EVDgyM"
      },
      "source": [
        "class BatchPadCollate:\n",
        "    def __init__(self, pad_idx):\n",
        "        self.pad_idx = pad_idx\n",
        "\n",
        "    def __call__(self, batch):\n",
        "\n",
        "        src_sentences = [item[0] for item in batch]\n",
        "        src_sentences = pad_sequence(src_sentences, batch_first= True, padding_value=self.pad_idx)\n",
        "\n",
        "        trg_sentences = [item[1] for item in batch]\n",
        "        trg_sentences = pad_sequence(trg_sentences, batch_first= True, padding_value=self.pad_idx) \n",
        "        \n",
        "        return src_sentences, trg_sentences "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UjZpgYpGE--j"
      },
      "source": [
        "## Transformer Network2 "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hUhYUI0ZE--k"
      },
      "source": [
        "class PositionalEncoding(nn.Module):\n",
        "    \n",
        "    def __init__(self,embed_dim, p_dropout, maximum_position_encoding):\n",
        "        super(PositionalEncoding, self).__init__()\n",
        "        \n",
        "        self.max_pos = maximum_position_encoding\n",
        "        self.dropout = nn.Dropout(p_dropout)\n",
        "        pos_enc = torch.zeros(self.max_pos, embed_dim)\n",
        "        positions = torch.arange(0, self.max_pos, dtype=torch.float).unsqueeze(1)\n",
        "        div_term = torch.exp(torch.arange(0, embed_dim, 2).float() * (-math.log(10000.0) / embed_dim))\n",
        "        pos_enc[:, 0::2] = torch.sin(positions * div_term)\n",
        "        pos_enc[:, 1::2] = torch.cos(positions * div_term)\n",
        "        pos_enc = pos_enc.unsqueeze(0).transpose(0, 1)\n",
        "        self.register_buffer('pos_enc', pos_enc)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = x + self.pos_enc[:x.size(0), :]\n",
        "        x = self.dropout(x)\n",
        "        \n",
        "        return x\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3eczGLQ9E--l"
      },
      "source": [
        "class _ScaledDotProductAttention(nn.Module):\n",
        "    \n",
        "    def __init__(self, embed_dim, device, p_dropout=0., kdim=None, vdim=None):\n",
        "        super(_ScaledDotProductAttention, self).__init__()\n",
        "        self.device = device\n",
        "        self.embed_dim = embed_dim\n",
        "        self.kdim = kdim if kdim is not None else embed_dim\n",
        "        self.vdim = vdim if vdim is not None else embed_dim\n",
        "\n",
        "        self.dropout = nn.Dropout(p_dropout)\n",
        "        self.softmax = nn.Softmax(dim=2)\n",
        "\n",
        "    def forward(self, query: Tensor, key: Tensor, value: Tensor, mask: Optional[Tensor] = None) -> Tuple[Tensor, Tensor]:\n",
        "          \n",
        "        \"\"\"\n",
        "        Calculate the attention weights.\n",
        "        q, k, v must have matching leading dimensions.\n",
        "        k, v must have matching penultimate dimension, i.e.: seq_len_k = seq_len_v.\n",
        "        The mask has different shapes depending on its type(padding or look ahead)\n",
        "        but it must be broadcastable for addition.\n",
        "\n",
        "        Args:\n",
        "            query : query shape == (..., seq_len_q, depth)\n",
        "            key : key shape == (..., seq_len_k, depth)\n",
        "            value: value shape == (..., seq_len_v, depth_v)\n",
        "            mask: Float tensor with shape broadcastable\n",
        "                to (..., seq_len_q, seq_len_k). Defaults to None.\n",
        "\n",
        "        Returns:\n",
        "            output, attention_weights\n",
        "        \"\"\"\n",
        "        qk = torch.matmul(query, torch.transpose(key,2,3))  # (..., seq_len_q, seq_len_k)\n",
        "        \n",
        "        # scale matmul_qk\n",
        "        scaled_attention_logits = qk / math.sqrt(self.kdim)\n",
        "\n",
        "        # add the mask to the scaled tensor.\n",
        "        if mask is not None:\n",
        "            scaled_attention_logits += (mask * -1e9)\n",
        "        \n",
        "        # softmax is normalized on the last axis (seq_len_k) so that the scores\n",
        "        # add up to 1.\n",
        "        attention_weights = self.softmax(scaled_attention_logits)  # (..., seq_len_q, seq_len_k)\n",
        "        \n",
        "        attention_weights = self.dropout(attention_weights)\n",
        "        \n",
        "        output = torch.matmul(attention_weights, value)  # (..., seq_len_q, depth_v)\n",
        "\n",
        "        return output"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A8aRZVaSE--m"
      },
      "source": [
        "class _MultiHeadAttention2(nn.Module):\n",
        "    \n",
        "    def __init__(self, embed_dim, num_heads, device, p_dropout=0.,\n",
        "                 kdim=None, vdim=None, epsilon = 1e-5):\n",
        "        super(_MultiHeadAttention2, self).__init__()\n",
        "        \n",
        "        self.device = device \n",
        "        self.embed_dim = embed_dim                               \n",
        "        self.num_heads = num_heads\n",
        "        self.kdim = kdim * num_heads if kdim is not None else embed_dim\n",
        "        self.vdim = vdim * num_heads if vdim is not None else embed_dim\n",
        "        self.d_model = self.vdim \n",
        "        self.head_dim = self.d_model // num_heads\n",
        "        assert self.head_dim * num_heads == self.d_model, \"embed_dim must be divisible by num_heads\"\n",
        "\n",
        "        self.Wq = nn.Linear(embed_dim, self.kdim,device = device)\n",
        "        self.Wk = nn.Linear(embed_dim, self.kdim,device = device)\n",
        "        self.Wv = nn.Linear(embed_dim, self.vdim,device = device)\n",
        "\n",
        "        self.self_attention = _ScaledDotProductAttention(self.head_dim, device,p_dropout, kdim, vdim)\n",
        "        self.layernorm = nn.LayerNorm((embed_dim,), eps = epsilon, device = device)\n",
        "        self.linear = nn.Linear(self.vdim, embed_dim,device = device)\n",
        "        self.dropout = nn.Dropout(p_dropout)\n",
        "\n",
        "    def split_heads(self, x, batch_size):\n",
        "        \"\"\"\n",
        "        Split the last dimension into (num_heads, depth).\n",
        "        Transpose the result such that the shape is (batch_size, num_heads, seq_len, depth)\n",
        "        \"\"\"\n",
        "        x = x.view((batch_size, -1, self.num_heads, self.head_dim))\n",
        "        return torch.transpose(x,1,2)\n",
        "\n",
        "    def forward(self, query: Tensor, key: Tensor, value: Tensor, mask: Optional[Tensor] = None) -> Tuple[Tensor, Tensor]:\n",
        "        \n",
        "        batch_size = query.shape[0]\n",
        "\n",
        "        q = self.Wq(query)  # (batch_size, seq_len, d_model)\n",
        "        k = self.Wk(key)    # (batch_size, seq_len, d_model)\n",
        "        v = self.Wv(value)  # (batch_size, seq_len, d_model)\n",
        "\n",
        "        q = self.split_heads(q, batch_size)  # (batch_size, num_heads, seq_len_q, depth)\n",
        "        k = self.split_heads(k, batch_size)  # (batch_size, num_heads, seq_len_k, depth)\n",
        "        v = self.split_heads(v, batch_size)  # (batch_size, num_heads, seq_len_v, depth)\n",
        "\n",
        "        # scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)\n",
        "        # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)\n",
        "        scaled_attention = self.self_attention(q, k, v, mask)\n",
        "\n",
        "        scaled_attention = torch.transpose(scaled_attention, 1,2)  # (batch_size, seq_len_q, num_heads, depth)\n",
        "\n",
        "        concat_attention = scaled_attention.reshape((batch_size, -1, self.d_model))  # (batch_size, seq_len_q, d_model)\n",
        "\n",
        "        output = self.linear(concat_attention)  # (batch_size, seq_len_q, d_model)\n",
        "\n",
        "        output = self.dropout(output)\n",
        "        output = self.layernorm(output)\n",
        "        return output"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sz1-fIL6E--n"
      },
      "source": [
        "class _PointwiseFeedicorward(nn.Module):\n",
        "    \n",
        "    def __init__(self, embed_dim, inner_dim,device, p_dropout=0.,epsilon = 1e-5):\n",
        "        super(_PointwiseFeedicorward, self).__init__()\n",
        "\n",
        "        self.linear1 = nn.Linear(embed_dim, inner_dim,device = device)\n",
        "        self.linear2 = nn.Linear(inner_dim, embed_dim,device = device)\n",
        "        self.layernorm = nn.LayerNorm((embed_dim,), eps = epsilon,device = device)\n",
        "        self.dropout = nn.Dropout(p_dropout)\n",
        "        self.relu = nn.ReLU()\n",
        "\n",
        "    def forward(self, x):\n",
        "        x1 = self.linear1(x)\n",
        "        x1 = self.relu(x1)\n",
        "        x1 = self.dropout(x1)\n",
        "        x1 = self.linear2(x1)\n",
        "        x1 = self.dropout(x1) \n",
        "        x1 = self.layernorm(x + x1)\n",
        "\n",
        "        return x1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6pLZwh4OE--o"
      },
      "source": [
        "class _EncoderLayer2(nn.Module):\n",
        "    \n",
        "    def __init__(self, embed_dim, inner_dim, num_heads, device, p_dropout = 0.,\n",
        "                 kdim = None, vdim = None,epsilon = 1e-5):\n",
        "        super(_EncoderLayer2, self).__init__()\n",
        "\n",
        "        self.device = device\n",
        "        self.mha = _MultiHeadAttention2(embed_dim, num_heads, device, p_dropout, kdim, vdim)\n",
        "        self.ff =  _PointwiseFeedicorward(embed_dim, inner_dim, device, p_dropout,epsilon = 1e-5)\n",
        "\n",
        "    def forward(self, x, mask = None):\n",
        "        \n",
        "        x = self.mha(x, x, x, mask)  # (batch_size, input_seq_len, d_model)\n",
        "        x = self.ff(x)         # (batch_size, input_seq_len, d_model)\n",
        "\n",
        "        return x"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0akVFNY8E--p"
      },
      "source": [
        "class _TransformerEncoder2(nn.Module):\n",
        "    \n",
        "    def __init__(self,num_layers, embed_dim,inner_dim, num_heads, device,\n",
        "                 p_dropout = 0., kdim = None, vdim = None, epsilon = 1e-5):\n",
        "        \n",
        "        super(_TransformerEncoder2, self).__init__()\n",
        "\n",
        "        self.num_layers = num_layers \n",
        "        self.device = device\n",
        "        \n",
        "        kwrgs = {'p_droput': p_dropout, 'kdim': kdim, 'vdim': vdim, 'epsilon': epsilon}\n",
        "        self.enc_layers = [_EncoderLayer2(embed_dim, inner_dim, num_heads,device,**kwargs) for _ in range(num_layers)]\n",
        "\n",
        "    def forward(self, x, mask):\n",
        "        \n",
        "        for i in range(self.num_layers):\n",
        "            x = self.enc_layers[i](x, mask)\n",
        "\n",
        "        return x  # (batch_size, input_seq_len, d_model)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gLy7YkMveTDA"
      },
      "source": [
        "class Transformer2(nn.Module):\n",
        "\n",
        "    def __init__(self, num_layers, embed_dim, ff_dim, num_heads, src_vocab_size,\n",
        "                 trg_vocab_size, device, p_dropout = 0., kdim = None,\n",
        "                 vdim = None, maximum_position_encoding = 5000, epsilon = 1e-5):\n",
        "        \n",
        "        super(Transformer2, self).__init__()\n",
        "\n",
        "        self.device = device\n",
        "        self.scale = embed_dim\n",
        "        self.src_embedding = nn.Embedding(src_vocab_size, embed_dim)\n",
        "        self.pos_encoder = PositionalEncoding(embed_dim, p_dropout, maximum_position_encoding)\n",
        "        kwrgs = {'p_droput': p_dropout, 'kdim': kdim, 'vdim': vdim, 'epsilon': epsilon}\n",
        "        self.transformer_encoder = _TransformerEncoder2(num_layers, embed_dim, ff_dim,\n",
        "                                                       num_heads,device = device, **kwargs)\n",
        "        \n",
        "\n",
        "        self.trg_embedding = nn.Embedding(trg_vocab_size, embed_dim)\n",
        "        self.pos_decoder = PositionalEncoding(embed_dim, p_dropout, maximum_position_encoding)\n",
        "        self.decoder_layer = TransformerDecoderLayer(embed_dim, num_heads, dim_feedforward = ff_dim,\n",
        "                                                     dropout = p_dropout, layer_norm_eps = epsilon, \n",
        "                                                     batch_first = True, device = device)\n",
        "        self.transformer_decoder = TransformerDecoder(self.decoder_layer, num_layers)\n",
        "\n",
        "\n",
        "        self.trg_classifier = nn.Linear(embed_dim, trg_vocab_size)\n",
        "\n",
        "    def make_padding_mask(self, seq, pad_idx):\n",
        "        seq_mask = seq == pad_idx\n",
        "        return seq_mask.to(self.device)\n",
        "    \n",
        "    def make_src_padding_mask(self, seq, pad_idx):\n",
        "        seq_mask = seq == pad_idx\n",
        "        return seq_mask.view(seq.shape[0],1,1,-1).to(self.device)\n",
        "\n",
        "    def generate_square_subsequent_mask(self, sz: int) -> Tensor:\n",
        "        r\"\"\"Generate a square mask for the sequence. The masked positions are filled with float('-inf').\n",
        "            Unmasked positions are filled with float(0.0).\n",
        "        \"\"\"\n",
        "        mask = (torch.triu(torch.ones(sz, sz)) == 1).transpose(0, 1)\n",
        "        mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))\n",
        "        return mask.to(self.device)\n",
        "\n",
        "    def forward(self, src, trg):\n",
        "        \n",
        "        src_padding_mask = self.make_src_padding_mask(src, 0)\n",
        "        mem_padding_mask = self.make_padding_mask(src, 0)\n",
        "        src = self.src_embedding(src).to(self.device)\n",
        "        src = self.pos_encoder(src).to(self.device)\n",
        "\n",
        "        enc_output = self.transformer_encoder(src, src_padding_mask) # (batch_size, src_seq_len, d_model)\n",
        "\n",
        "\n",
        "        trg_padding_mask = self.make_padding_mask(trg, 0)\n",
        "        trg_mask = self.generate_square_subsequent_mask(trg.shape[1])\n",
        "        kwargs = {'tgt_mask': trg_mask, 'tgt_key_padding_mask': trg_padding_mask, 'memory_key_padding_mask': mem_padding_mask}\n",
        "        trg = self.trg_embedding(trg).to(self.device)\n",
        "        trg = self.pos_decoder(trg).to(self.device)\n",
        "\n",
        "        dec_output = self.transformer_decoder(trg, enc_output,**kwargs) # (batch_size, trg_seq_len, d_model)\n",
        "\n",
        "\n",
        "        final_output = self.trg_classifier(dec_output) # (batch_size, trg_seq_len, trg_vocab_size)\n",
        "\n",
        "        return final_output\n",
        "    def encode(self, src, mask):\n",
        "        src = self.src_embedding(src).to(self.device) * math.sqrt(self.scale)\n",
        "        src = self.pos_encoder(src)\n",
        "\n",
        "        enc_output = self.transformer_encoder(src, mask) # (batch_size, src_seq_len, d_model)\n",
        "        return enc_output\n",
        "    \n",
        "    def decode(self, trg, src_enc, trg_mask, mem_padding_mask):\n",
        "        kwargs = {'tgt_mask': trg_mask,'memory_key_padding_mask': mem_padding_mask}\n",
        "        \n",
        "        trg = self.trg_embedding(trg).to(self.device) * math.sqrt(self.scale)\n",
        "        trg = self.pos_decoder(trg)\n",
        "        # dec_output.shape == (batch_size, trg_seq_len, d_model)\n",
        "        dec_output = self.transformer_decoder(trg, src_enc, **kwargs)\n",
        "        final_output = self.trg_classifier(dec_output) # (batch_size, trg_seq_len, trg_vocab_size)\n",
        "\n",
        "        return final_output\n",
        "\n",
        "    def translate_sentence(self,src,max_len, sos_idx, eos_idx):\n",
        "        src = src.to(self.device)\n",
        "        src_padding_mask = self.make_src_padding_mask(src, 0)\n",
        "        mem_padding_mask = self.make_padding_mask(src, 0)\n",
        "\n",
        "        enc_src = self.encode(src, src_padding_mask)\n",
        "\n",
        "        trg_indexes = [sos_idx]\n",
        "\n",
        "        for i in range(max_len):\n",
        "\n",
        "            trg = torch.LongTensor(trg_indexes).unsqueeze(0).to(device)\n",
        "            trg_mask = self.generate_square_subsequent_mask(trg.shape[1])\n",
        "\n",
        "            output = self.decode(trg, enc_src, trg_mask, mem_padding_mask)\n",
        "            \n",
        "            pred_token = output.argmax(2)[:,-1].item()\n",
        "            \n",
        "            trg_indexes.append(pred_token)\n",
        "\n",
        "            if pred_token == eos_idx:\n",
        "                break\n",
        "        \n",
        "        return trg_indexes"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "getLlc8Tdy9l"
      },
      "source": [
        "## Training "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "f5db7c85a6c248db93599255e9fe5ab2",
            "5bb4b5aefb9742568ae9ba9854972c9f",
            "0b56d466f92741399dc0b8389b088221",
            "4ab2f72494214c03b1fd81d794c930b5",
            "131bf4b442624ced8bd05761e6ab896e",
            "bd4d97e6e22b4cb0a33bb02f8914f161",
            "e03ad45e77a3478c9ea067a51db6cc9c",
            "f903c3bd58484b268d8ac5c091706f3d",
            "f63ca6a23e944ea9bdb79b2bc55a4c98",
            "08f51be517ac4977a31a2bc8c1871ef1",
            "05827341f76e44ceb4d42b4f2fa3817f",
            "382c798e36d64b80b61345c002649069",
            "eec3e32e32ec488fa025cbea483a104f",
            "591e3e1dda174753933f319b577c6e06",
            "0fe9f29e9b974ab6b5d3905ab0b67caf",
            "19d477efe05346188329ae246d2ca8d8",
            "b19e1f098c4843968466974ca834ae5c",
            "e7ed6a7ba2ac440b949f34c500ba88c6",
            "bda8cd154c5747779eee8f8b2510ce21",
            "339ce019bd084de78db5a4c25340ec9b",
            "87c6b4b1abf84065afbd937b13c7cbb8",
            "2a9d2b6dc9f740fdb1f223f69b533bcc",
            "3f993e1d04264feda6814b3c3c5da36c",
            "c0baa3849b5e45fbac079093fd9a9cdc",
            "b4064ba3432241a69f6333ae5f8160c4",
            "8e68382140744142a765b887d6c18419",
            "ddcbd7c0491a4b17b149f39f5b52cb5f",
            "298cd16df4304e9588d3564a840beb97",
            "d6a2e037449845c3bd289fad6cac9032",
            "51ba654e3a774d81866b13eb7c32cc4e",
            "e9c3e524ecf24d09a35a22af93fccc1c",
            "07724286976f4de4bdfe743da33d351c",
            "060446b4b2704ae6ba6345ddb927652a",
            "b755c5560feb49fc8b9c86b943204f18",
            "9174015ff178401ab20ec71542e2f877",
            "a8321c54e8954ff0823c60ff37fa73dd",
            "e228b4f804074906b97f49fe822e139f",
            "c2d68d22e0ad4a5fb568d3cde0538e2c",
            "89c1233d247d459dbd742bfa31c8c7e6",
            "e49e6d128cd741408464c09d33b0ece6",
            "d3c8832fe0074a7f9d9cd47033b2ad0c",
            "b414d48255fe49198e5d0e730fe51dc3",
            "14292ae94ba04df38bd45c30f5878764",
            "77d9348493a74fd6937af2bb5b824e78",
            "29ec7ae02242461086ec54342cf18097",
            "a394fb6d37a242c993b171e6a21b5d0a",
            "717550382c5c483d82e9c111464e8f5d",
            "be254a6597f64648bb914e05f6927bea",
            "84eff2667dc54a019c9c1e7174efccab",
            "2619cc8a923b4fd8ac8973f578409a0f",
            "c2c0b01a123b417bb15dec8f774014b6",
            "91ed93c580e242ff924acc956ac0ed07",
            "90e4a3f0f8384fec92da55208e87644f",
            "7861ad1a1cb54c438c35785e7d89a266",
            "eac2e9190ba94914a9f10d54e5553b45",
            "b8fb0db1486b4b9cbf96a4af24730df1",
            "d17f501095f74ab9a7e0ed7a6314d3d4",
            "f3d656e2180e4e979db84258f396e8fb",
            "7ed2b3238d2e452e8e55384a43c1cd04",
            "b47c256c78424157a744614ab6ff7e04",
            "93a5c973137c4b91af32c4b317d9ac53",
            "a5a27616970840688cbd071686922197",
            "7d8760ee5a834b1397d4a4fc6cdb70c5",
            "c9f834a8ccac432ca127ec0beab18838",
            "0e9c111a8e9a4aad87e1a01d4e6fdac8",
            "6591b01a42d24ea185309c6104d90639",
            "a657a985de41458a983c718539567227",
            "8b0cf3c8426c404696175ef1ea747a9d",
            "bb0263a7717d49619a2895660bae3bf2",
            "cefbc4dfed1e47e29396d9ed832a2f4d",
            "5fad9074c5d1412399b3173a9eafb71a",
            "61e18faaaae94946a6370d77822e0740",
            "a57989aad9db4e22b31a8868aaef5e94",
            "a363a6702f924397ac2ad070c19bdecc",
            "9b545c5e253d40fdaca01f3ed90ff7bc",
            "01a13c5a74c74f588a797bbfe81d64a7",
            "aa069744861e4ff8880bf42f3acd1210",
            "7f0bb8185e96453198e5a4211e18e79e",
            "f22823efc7884a0dba66f3cdf9b5ae77",
            "d956ae9165154fed94546105401d648d"
          ]
        },
        "id": "Lk_Gy5bWdy9m",
        "outputId": "edf2437f-d19a-46f8-d84f-e9d31a56a6e2"
      },
      "source": [
        "torch.manual_seed(2045)\n",
        "\n",
        "pad_idx = 0\n",
        "\n",
        "# Parameters\n",
        "params = {'batch_size': 128,\n",
        "          'shuffle': True,\n",
        "          'num_workers':2,\n",
        "          'collate_fn' : BatchPadCollate(pad_idx=pad_idx)\n",
        "          }\n",
        "\n",
        "# Training and validation Data Generators\n",
        "training_set = AFEC_Dataset(en_train_dic,fa_train_dic,partition['train'],en_vocab_dic, fa_vocab_dic)\n",
        "training_generator = torch.utils.data.DataLoader(training_set, **params)\n",
        "\n",
        "validation_set =  AFEC_Dataset(en_train_dic,fa_train_dic,partition['validation'],en_vocab_dic, fa_vocab_dic)\n",
        "validation_generator = torch.utils.data.DataLoader(validation_set, **params)\n",
        "\n",
        "# Hyperparameters\n",
        "embed_dim = 256\n",
        "num_heads = 8\n",
        "dim_key = 64\n",
        "dim_value = 64\n",
        "dim_inner = 1024\n",
        "num_layers = 3\n",
        "src_vocab_size = len(en_vocab_dic.get_str_2_idx())\n",
        "trg_vocab_size = len(fa_vocab_dic.get_str_2_idx())\n",
        "learning_rate = 1e-4\n",
        "num_iterations = 5000\n",
        "\n",
        "kwargs = {'p_dropout': 0.1,\n",
        "          'kdim': dim_key,\n",
        "          'vdim':dim_value,\n",
        "          }\n",
        "# Model\n",
        "# model3 = Transformer2(num_layers, embed_dim, dim_inner, num_heads, src_vocab_size,trg_vocab_size, device,**kwargs).to(device)\n",
        "\n",
        "# Define Loss function and Optimizer\n",
        "criterion = nn.CrossEntropyLoss(ignore_index = pad_idx)\n",
        "optimizer = optim.Adam(model3.parameters(), lr = learning_rate, betas=(0.9, 0.98), eps=1e-9)\n",
        "\n",
        "# train_losses3 = []\n",
        "# validation_losses3 = []\n",
        "\n",
        "torch.autograd.set_detect_anomaly(True)\n",
        "\n",
        "train_dataiter = iter(training_generator)\n",
        "\n",
        "trainloss, valloss, running_trainloss, running_valloss = 0.0, 0.0, 0.0, 0.0\n",
        "\n",
        "# Loop over epochs\n",
        "for itr in tqdm(range(num_iterations)):\n",
        "\n",
        "    # Training        \n",
        "    batch_train_src, batch_train_trg = train_dataiter.next()\n",
        "\n",
        "    # Transfer to GPU\n",
        "    batch_train_src, batch_train_trg = batch_train_src.to(device), batch_train_trg.to(device)\n",
        "\n",
        "    # zero the parameter gradients\n",
        "    optimizer.zero_grad()\n",
        "    \n",
        "    # Model computations\n",
        "    # forward + backward + optimize\n",
        "    batch_train_outputs = model3(batch_train_src, batch_train_trg[: , :-1])\n",
        "    \n",
        "    loss = criterion(torch.transpose(batch_train_outputs,1,2), batch_train_trg[: , 1:])\n",
        "\n",
        "    loss.backward()\n",
        "\n",
        "    # Clip to avoid exploding gradient issues, makes sure grads are within a healthy range\n",
        "    nn.utils.clip_grad_norm_(model3.parameters(), max_norm=1,error_if_nonfinite= False)\n",
        "\n",
        "    optimizer.step()\n",
        "\n",
        "    # print statistics\n",
        "    trainloss += loss.item()\n",
        "    \n",
        "    # Validation\n",
        "    if itr % 500 == 499:\n",
        "        with torch.set_grad_enabled(False):\n",
        "            val_cnt = 1\n",
        "            for batch_src, batch_trg in tqdm(validation_generator):\n",
        "                # Transfer to GPU\n",
        "                batch_src, batch_trg = batch_src.to(device), batch_trg.to(device)\n",
        "                \n",
        "                # Model computations\n",
        "                # forward + backward + optimize\n",
        "                batch_outputs = model3(batch_src, batch_trg[: , :-1])\n",
        "                \n",
        "                val_loss = criterion(torch.transpose(batch_outputs,1,2), batch_trg[: , 1:])\n",
        "\n",
        "                valloss += val_loss.item()\n",
        "                val_cnt += 1\n",
        "\n",
        "        print(\"Cost of Train data after %i iterations : %f\" %(itr + 1, trainloss / 500))\n",
        "        print(\"Cost of Validation data after %i iterations: %f\" %(itr + 1, valloss / val_cnt))\n",
        "        running_valloss = 0\n",
        "\n",
        "        train_losses3.append(trainloss/500)\n",
        "        trainloss = 0\n",
        "        validation_losses3.append(valloss/val_cnt)\n",
        "        valloss = 0 "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f5db7c85a6c248db93599255e9fe5ab2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=5000.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f63ca6a23e944ea9bdb79b2bc55a4c98",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 500 iterations : 4.897921\n",
            "Cost of Validation data after 500 iterations: 4.944427\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b19e1f098c4843968466974ca834ae5c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 1000 iterations : 4.890292\n",
            "Cost of Validation data after 1000 iterations: 4.932391\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b4064ba3432241a69f6333ae5f8160c4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 1500 iterations : 4.881585\n",
            "Cost of Validation data after 1500 iterations: 4.919564\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "060446b4b2704ae6ba6345ddb927652a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 2000 iterations : 4.875365\n",
            "Cost of Validation data after 2000 iterations: 4.912429\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d3c8832fe0074a7f9d9cd47033b2ad0c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 2500 iterations : 4.869326\n",
            "Cost of Validation data after 2500 iterations: 4.902979\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "84eff2667dc54a019c9c1e7174efccab",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 3000 iterations : 4.855814\n",
            "Cost of Validation data after 3000 iterations: 4.892675\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d17f501095f74ab9a7e0ed7a6314d3d4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 3500 iterations : 4.859946\n",
            "Cost of Validation data after 3500 iterations: 4.885011\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0e9c111a8e9a4aad87e1a01d4e6fdac8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 4000 iterations : 4.847824\n",
            "Cost of Validation data after 4000 iterations: 4.879061\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a57989aad9db4e22b31a8868aaef5e94",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 4500 iterations : 4.841477\n",
            "Cost of Validation data after 4500 iterations: 4.871543\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "StopIteration",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mStopIteration\u001b[0m                             Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-61-77b0f2e7ae88>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m     \u001b[0;31m# Training\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 55\u001b[0;31m     \u001b[0mbatch_train_src\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_train_trg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_dataiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     56\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m     \u001b[0;31m# Transfer to GPU\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    519\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sampler_iter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    520\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 521\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    522\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    523\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1174\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_persistent_workers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1175\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_shutdown_workers\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1176\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1177\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1178\u001b[0m             \u001b[0;31m# Now `self._rcvd_idx` is the batch index we want to fetch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mStopIteration\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 246,
          "referenced_widgets": [
            "9bfaa34311bd4b3798684b14172ad13c",
            "c12a585da7fe49a5ba7e5463af4ea883",
            "1b94f0316cf14eebae38874120950eb0",
            "58d21a3330404c8bac11bfc403d875e3",
            "34d59c83f72d4fd481918e011d14d424",
            "747a21b4ec41462799a7a25072c8ccb3",
            "1028f59cac6f4f3c820c6f651f0f5981",
            "136085f60c9342649c6d549bf65a98d4",
            "5618bad07c304b4f94df5791bd8b6977",
            "44dae7da0cb9416dafbbb26ee8ce7dbb",
            "37402bb9fe13498e9cafbff33850f97b",
            "b45dbf7e48fa44e3b2ff5e80e7e218f3",
            "aaa25f5cdd364e9caba25621be04ef94",
            "af8c341667884f8b8c515644f5edb5c0",
            "f4e5f576be2147f5964e44a4f3254bed",
            "13fd768da52b45fca78a7aef449692ed",
            "ac3660b032cb44729cb33af38a2625f7",
            "cc43ea9c3a604d83808ff62bafb84127",
            "a2777c6c74f4404e8b2c08ceaff60bcf",
            "aaec11c763f34a5aabebeaceca0e8c3b",
            "3997def9504c46199905c520c88dcd5c",
            "d67e5af4aac44b68b0eb3a0d83804614",
            "1ea0d51e78564f0ca4dbda6288e8b4df",
            "336c8ee34b4141f1862a119717c549d7",
            "0b989c116f6b4b5fb77f379acfc58db5",
            "4432747017df47659621e3662d1d7bb6",
            "ad6eb2ae8c9043c38385fb47524abf89",
            "f89cde0b8ee6441b8bfe362ca38958b9",
            "3e4efe1c4cf541b5b780619d470d5379",
            "37ccaf635ec94cbd8fe4f3ad51909fab",
            "f00d59fb2ad545188ec2448af944ed8e",
            "936f3cb2ebb34a108f3aa97daa709a9a"
          ]
        },
        "id": "QYwXmclyHoBq",
        "outputId": "954d2d48-bdcc-48f0-c9a3-0e1ff3cdedc0"
      },
      "source": [
        "train_dataiter = iter(training_generator)\n",
        "\n",
        "trainloss, valloss, running_trainloss, running_valloss = 0.0, 0.0, 0.0, 0.0\n",
        "\n",
        "# Loop over epochs\n",
        "for itr in tqdm(range(num_iterations)):\n",
        "\n",
        "    # Training        \n",
        "    batch_train_src, batch_train_trg = train_dataiter.next()\n",
        "\n",
        "    # Transfer to GPU\n",
        "    batch_train_src, batch_train_trg = batch_train_src.to(device), batch_train_trg.to(device)\n",
        "\n",
        "    # zero the parameter gradients\n",
        "    optimizer.zero_grad()\n",
        "    \n",
        "    # Model computations\n",
        "    # forward + backward + optimize\n",
        "    batch_train_outputs = model3(batch_train_src, batch_train_trg[: , :-1])\n",
        "    \n",
        "    loss = criterion(torch.transpose(batch_train_outputs,1,2), batch_train_trg[: , 1:])\n",
        "\n",
        "    loss.backward()\n",
        "\n",
        "    # Clip to avoid exploding gradient issues, makes sure grads are within a healthy range\n",
        "    nn.utils.clip_grad_norm_(model3.parameters(), max_norm=1,error_if_nonfinite= False)\n",
        "\n",
        "    optimizer.step()\n",
        "\n",
        "    # print statistics\n",
        "    trainloss += loss.item()\n",
        "    \n",
        "    # Validation\n",
        "    if itr % 500 == 499:\n",
        "        with torch.set_grad_enabled(False):\n",
        "            val_cnt = 1\n",
        "            for batch_src, batch_trg in tqdm(validation_generator):\n",
        "                # Transfer to GPU\n",
        "                batch_src, batch_trg = batch_src.to(device), batch_trg.to(device)\n",
        "                \n",
        "                # Model computations\n",
        "                # forward + backward + optimize\n",
        "                batch_outputs = model3(batch_src, batch_trg[: , :-1])\n",
        "                \n",
        "                val_loss = criterion(torch.transpose(batch_outputs,1,2), batch_trg[: , 1:])\n",
        "\n",
        "                valloss += val_loss.item()\n",
        "                val_cnt += 1\n",
        "\n",
        "        print(\"Cost of Train data after %i iterations : %f\" %(itr + 1, trainloss / 500))\n",
        "        print(\"Cost of Validation data after %i iterations: %f\" %(itr + 1, valloss / val_cnt))\n",
        "        running_valloss = 0\n",
        "\n",
        "        train_losses3.append(trainloss/500)\n",
        "        trainloss = 0\n",
        "        validation_losses3.append(valloss/val_cnt)\n",
        "        valloss = 0 "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9bfaa34311bd4b3798684b14172ad13c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=5000.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5618bad07c304b4f94df5791bd8b6977",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 500 iterations : 4.879543\n",
            "Cost of Validation data after 500 iterations: 4.925639\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ac3660b032cb44729cb33af38a2625f7",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Cost of Train data after 1000 iterations : 4.876445\n",
            "Cost of Validation data after 1000 iterations: 4.919487\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0b989c116f6b4b5fb77f379acfc58db5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=535.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "7Xf0PTM3dy9q",
        "outputId": "a31d9a0e-665a-4774-f6e9-630cdb4d8c3d"
      },
      "source": [
        "# plot the cost\n",
        "plt.plot(range(500,27500,500),train_losses3)\n",
        "plt.plot(range(500,27500,500),validation_losses3)\n",
        "plt.title(\"Cost of Transfromer vs iters \")\n",
        "plt.ylabel('Cost')\n",
        "plt.xlabel('Iterations')\n",
        "plt.legend(labels = [\"Train\",\"Validation\"])\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU5fX48c+ZrGQnG0vYN1FkjyCCCm5VxKVWVKpFSqvir7W1trVqW7Xafmurti5tte7VWrFVwX3BBUVBZQvIvskSyEZCkknIPuf3x70JQ0zClskkmfN+ve5r7txtzjOTzJnnufd5rqgqxhhjQpcn2AEYY4wJLksExhgT4iwRGGNMiLNEYIwxIc4SgTHGhDhLBMYYE+IsEZh2R0QmishmESkTkYuDHU9jItJNRD4REa+I3B/seNqCiPRxP4+wYMdiWp8lghAnIt8VkWXuP3mOiLwtIpOO8ZjbReSsYzjEXcDfVDVOVec3OnaZ3+QTkQq/51ceS9xH4FpgL5Cgqj9vo9cMKlXd6X4edQAislBEfhjsuEzrsEQQwkTkJuAB4P+AbkAf4B/ARcGMC+gLrG1qhftlFKeqccBO4AK/Zc/Xbyci4QGOb5020xszwK/domC+9pGwmkU7o6o2heAEJAJlwPQWtonCSRR73OkBIMpdlwq8ARQDRcAinB8WzwE+oMI9/s3NHPsaYIu772tAT3f51kb7R7UQ33bgLHd+MpAN/ArIdePo6sZYAOxz53v57b8QuBv4DPAC7wGp7rpo4N9AoVvGpTjJ8hmgBqh24zsLuBN4yd2+FPgh0NMtV5Fbzmv8XvdO4H/u9l7gK2AIcCuQD+wCzmn0WT0J5AC7gd8DYe66WW78f3Vj/X2j96in+14m+y0bjVOjiQAGAR8DJe6yF5t5r/sBCoQDfwDqgEr3Pfibu81QYIFb5o3AZX77PwM8ArwFlLvv21Rgnfse7AZ+Eez/i1Cdgh6ATUH64OFcoBYIb2Gbu4DPgXQgDVgM3O2u+yPwqPtlEgGcCoi7bjvuF3Qzxz3D/dIZg5NsHgY+8Vvf4v5NbYeTCGqBP7nH7AKkAN8BYoB498t3vt/+C3ESzxB3+4XAPe6664DX3X3DgLE4TUH1X2q/9zvOnTjJ4WKcZNgF+ASndhUNjMJJRmf4bV8JfMv9Yn0W+Br4tfteXgN87Xf8ecA/gVj3s/gSuM5dN8st9w3usbo08T59yMGJ6F7gUXf+Bfd1PW6sk5p5r/vhJgK/9+6HfutjcRLY99046pPNCX7vWQkw0e+1coBT3fVdgTHB/r8I1cmahkJXCrBXVWtb2OZK4C5VzVfVAuB3wPfcdTVAD6Cvqtao6iJ1/6MPw5XAU6q6QlWrcH4JTxCRfkdTED8+4A5VrVLVClUtVNWXVXW/qnpxfsme3mifp1V1k6pWAP/F+dKuL18KMEhV61R1uaqWtvDaS1R1vqr6cGpLE4FfqWqlqmYBTwAz/bZfpKrvuu///3AS7T2qWgPMBfqJSJKIdMP55Xyjqparaj7Or/8r/I61R1UfVtVatxyN/QeYASAi4u77H79y9sWpkVWq6qctlLEl04Dtqvq0G8dK4GVgut82r6rqZ6rqU9VK97VPEJEEVd2nqiuO8rXNMbJEELoKgdRDtCn3BHb4Pd/hLgPnV+UW4D0R2SYitxzBax90XFUtc+PJOIJjNKXA/YIBQERiROSfIrJDREpxfqUnNWqfzvWb3w/EufPPAe8Cc0Vkj4j8WUQiWnjtXX7zPYEiN/nU28HB5cvzm6/AScp1fs9xY+mLU0vIEZFiESnGqR2kN/PaTXkZJ9H2AE7DSZiL3HU3AwJ8KSJrRWT2IY7VnL7A+PoY3TivBLq3EOd3cJLcDhH5WEQmHOVrm2NkiSB0LQGqcJozmrMH5x+8Xh93GarqVdWfq+oA4ELgJhE5093uUDWDg44rIrE4v753H1EJvqnx6/4cOA4Yr6oJOF+C4HzxtXwgp5bzO1U9ATgF5xfvzJZ28ZvfAySLSLzfsj4cXfl24XxOqaqa5E4Jqjqsmdf+ZmCq+3DOf1wOfBeYW197U9VcVb1GVXviNIf9Q0QGHUZcjV9zF/CxX4xJ6pzAv765fVR1qapehJPU5uPUyEwQWCIIUapaAtwO/F1ELnZ/PUeIyHki8md3sxeA34hImoikutv/G0BEponIILepoQTn5KHP3S8PGNDCy78AfF9ERolIFM5VS1+o6vZWLmY8zq/rYhFJBu443B1FZIqIDHdrD6U4zRi+Q+wGgKruwjmf8kcRiRaREcAPcN+7I6GqOThf4veLSIKIeERkoIg0buI6lP/gJLJLOdAshIhMF5Fe7tN9OF/Wh1POxp/xG8AQEfme+3cUISInicjxTe0sIpEicqWIJLrNYaWH+bomACwRhDBVvR+4CfgNzsnMXcCPcX6dgXN1yjJgNc6VLSvcZQCDgfdxrhpZAvxDVT9y1/0RJ4EUi8gvmnjd94Hf4jRZ5AADObjNu7U8gHPidi/OSe93jmDf7jhXApUC63GurHnuCPafgXOCdQ/Oyd473HIfjZlAJM4VNvvcuHoc4TFew/nMclV1ld/yk4AvRKTM3eanqrrtMI73IHCpiOwTkYfcZrBzcD7HPThNbvUn7pvzPWC722w3B6cpyQRB/VUexhhjQpTVCIwxJsRZIjDGmBBnicAYY0KcJQJjjAlxHWKAKn+pqanar1+/YIdhjDEdyvLly/eqalpT6zpcIujXrx/Lli0LdhjGGNOhiMiO5tZZ05AxxoQ4SwTGGBPiLBEYY0yI63DnCIwxnUdNTQ3Z2dlUVlYeemNzWKKjo+nVqxcRES0NlnswSwTGmKDJzs4mPj6efv364YxfaI6FqlJYWEh2djb9+/c/7P2sacgYEzSVlZWkpKRYEmglIkJKSsoR17AsERhjgsqSQOs6mvczZBLBhtxS/vzOBor3Vwc7FGOMaVdCJhHsKNzPPxZuJXtfU7d0NcaEosLCQkaNGsWoUaPo3r07GRkZDc+rq1v+0bhs2TJ+8pOftFGkgRUyJ4u7JUQDkFdayYkZiUGOxhjTHqSkpJCVlQXAnXfeSVxcHL/4xYF7KdXW1hIe3vTXZGZmJpmZmW0SZ6CFTI0gPd65UVK+tyrIkRhj2rNZs2YxZ84cxo8fz80338yXX37JhAkTGD16NKeccgobN24EYOHChUybNg1wksjs2bOZPHkyAwYM4KGHHgpmEY5YyNQI0txEkFdq1ysb0x797vW1rNtT2qrHPKFnAndcMOyI98vOzmbx4sWEhYVRWlrKokWLCA8P5/333+e2227j5Zdf/sY+GzZs4KOPPsLr9XLcccdx/fXXH9G1/MEUMokgIsxDSmwkeaVWIzDGtGz69OmEhYUBUFJSwtVXX83mzZsREWpqaprc5/zzzycqKoqoqCjS09PJy8ujV69ebRn2UQuZRACQnhBNgddqBMa0R0fzyz1QYmNjG+Z/+9vfMmXKFObNm8f27duZPHlyk/tERUU1zIeFhVFbWxvoMFtNyJwjAOc8gdUIjDFHoqSkhIyMDACeeeaZ4AYTICGVCLolRJFvNQJjzBG4+eabufXWWxk9enSH+pV/JERVA3NgkeOAF/0WDQBuV9UH/LYR4EFgKrAfmKWqK1o6bmZmph7tjWnuf28jf/9oC5v/MJUwj/VmNCbY1q9fz/HHHx/sMDqdpt5XEVmuqk1e7xqwcwSquhEY5QYQBuwG5jXa7DxgsDuNBx5xHwMiPT4Kn0JheRXp8dGBehljjOlQ2qpp6Exgq6o2vlXaRcCz6vgcSBKRHoEKIt3tVJZv5wmMMaZBWyWCK4AXmlieAezye57tLjuIiFwrIstEZFlBQcFRB3GgU5mdJzDGmHoBTwQiEglcCPzvaI+hqo+paqaqZqalpR11LAeGmbAagTHG1GuLGsF5wApVzWti3W6gt9/zXu6ygEiNs97FxhjTWFskghk03SwE8BowUxwnAyWqmhOoQCLDnd7FNt6QMcYcENBEICKxwNnAK37L5ojIHPfpW8A2YAvwOPD/AhkPOGMO5VuNwBgDTJkyhXffffegZQ888ADXX399k9tPnjyZ+svXp06dSnFx8Te2ufPOO7nvvvtafN358+ezbt26hue3334777///pGG32oCmghUtVxVU1S1xG/Zo6r6qDuvqvojVR2oqsNV9eg6CByBbgnRViMwxgAwY8YM5s6de9CyuXPnMmPGjEPu+9Zbb5GUlHRUr9s4Edx1112cddZZR3Ws1hBSPYvB6V1s5wiMMQCXXnopb775ZsNNaLZv386ePXt44YUXyMzMZNiwYdxxxx1N7tuvXz/27t0LwB/+8AeGDBnCpEmTGoapBnj88cc56aSTGDlyJN/5znfYv38/ixcv5rXXXuOXv/wlo0aNYuvWrcyaNYuXXnoJgA8++IDRo0czfPhwZs+eTVVVVcPr3XHHHYwZM4bhw4ezYcOGVnsfQmrQOYD0+Gj2llVT51PrXWxMe/L2LZD7Veses/twOO+eZlcnJyczbtw43n77bS666CLmzp3LZZddxm233UZycjJ1dXWceeaZrF69mhEjRjR5jOXLlzN37lyysrKora1lzJgxjB07FoBLLrmEa665BoDf/OY3PPnkk9xwww1ceOGFTJs2jUsvvfSgY1VWVjJr1iw++OADhgwZwsyZM3nkkUe48cYbAUhNTWXFihX84x//4L777uOJJ55ojXcpNGsEdT6lsNyah4wxBzcP1TcL/fe//2XMmDGMHj2atWvXHtSM09iiRYv49re/TUxMDAkJCVx44YUN69asWcOpp57K8OHDef7551m7dm2LsWzcuJH+/fszZMgQAK6++mo++eSThvWXXHIJAGPHjmX79u1HW+RvCLkaQVr8gd7FNsyEMe1IC7/cA+miiy7iZz/7GStWrGD//v0kJydz3333sXTpUrp27cqsWbOorDy65uRZs2Yxf/58Ro4cyTPPPMPChQuPKdb6oa5be5jrkKwRgPUuNsY44uLimDJlCrNnz2bGjBmUlpYSGxtLYmIieXl5vP322y3uf9pppzF//nwqKirwer28/vrrDeu8Xi89evSgpqaG559/vmF5fHw8Xq/3G8c67rjj2L59O1u2bAHgueee4/TTT2+lkjYv5BJBuvUuNsY0MmPGDFatWsWMGTMYOXIko0ePZujQoXz3u99l4sSJLe47ZswYLr/8ckaOHMl5553HSSed1LDu7rvvZvz48UycOJGhQ4c2LL/iiiu49957GT16NFu3bm1YHh0dzdNPP8306dMZPnw4Ho+HOXPmEGgBG4Y6UI5lGGqA6lofQ37zNj87awg/PWtwK0ZmjDlSNgx1YBzpMNQhVyOIDPeQHBtJnjUNGWMMEIKJAJxRSG0oamOMcYRkInB6F1uNwJj2oKM1T7d3R/N+hmQisBqBMe1DdHQ0hYWFlgxaiapSWFhIdPSRXRofcv0IwKkRFJRVWe9iY4KsV69eZGdncyw3nDIHi46OplevXke0T0gmgnS3d3FReTVp7l3LjDFtLyIigv79+wc7jJAXok1D9X0J7DyBMcaEZiKw3sXGGNMgJBNB/b2L7YSxMcaEaCJIa7h3sSUCY4wJyURQ37vYmoaMMSZEEwE4fQmsRmCMMaGcCBKiKbAagTHGhG4i6GY1AmOMAUI4EaQnRFFQVoXPZ13bjTGhLWQTQbeEaPfexdXBDsUYY4IqZBNBenz9JaR2nsAYE9pCNxG4ncoKvHaewBgT2kI2EXRLsPGGjDEGQjgR1PcuzrcagTEmxIVsImi4d7HVCIwxIS50EkHOanj9Rqgub1iUHh9lNQJjTMgLnURQlg/Ln4bspQ2L0hOiybcagTEmxIVOIug9DsQDO5Y0LLIagTHGhFIiiE6A7sNhx2cNi7olOInAehcbY0JZ6CQCgD6nOE1DtU5vYutdbIwxoZYI+p4CtZWQkwUc6F1s9yUwxoSygCYCEUkSkZdEZIOIrBeRCY3WTxaREhHJcqfbAxkPfdyXd5uH0u2WlcYYQ3iAj/8g8I6qXioikUBME9ssUtVpAY7DEZcGqUOcE8aTfmY1AmOMIYA1AhFJBE4DngRQ1WpVLQ7U6x22PhNg5+fgqyMt3u5dbIwxgWwa6g8UAE+LyEoReUJEYpvYboKIrBKRt0VkWFMHEpFrRWSZiCwrKCg4tqj6ToSqEshfR1R4GF1jIqxGYIwJaYFMBOHAGOARVR0NlAO3NNpmBdBXVUcCDwPzmzqQqj6mqpmqmpmWlnZsUfWtP0+wGHCuHLIagTEmlAUyEWQD2ar6hfv8JZzE0EBVS1W1zJ1/C4gQkdQAxgRJfSCxd0MiSLNOZcaYEBewRKCqucAuETnOXXQmsM5/GxHpLiLizo9z4ykMVEwN+p7iJAJVuiVEs6e4AlXrVGaMCU2B7kdwA/C8iKwGRgH/JyJzRGSOu/5SYI2IrAIeAq7QtvhG7jMByvOhaBuj+yRR4K1ia0FZwF/WGGPao4BePqqqWUBmo8WP+q3/G/C3QMbQpL4TnccdnzHluOkAfLghn0Hp8W0eijHGBFto9SyulzoYYlJhxxJ6JnVhaPd4PlifH+yojDEmKEIzEYg4Vw+5PYzPGJrOsh37KKmoCXJgxhjT9kIzEYAzAF3xDijZzZnHp1PnUxZtPsY+CsYY0wGFbiLoe4rzuHMJo3p3pWtMBB9a85AxJgSFbiLoPhwi42HHYsI8wulD0li4qYA6uzeBMSbEhG4i8IRBn/ENHcumDE2nqLyaVdnBHw7JGGPaUugmAnD6ExSsh/1FnD4kjTCPWPOQMSbkhHYiqO9PsHMJSTGRjO3TlQ83WCIwxoSW0E4EGWMgLOqg5qF1OaXklthopMaY0BHaiSA8CnplHtSfAOCjjVYrMMaEjtBOBAADp8CelbBvB0O6xZGR1MV6GRtjQoolghFXAAKrXkBEOGNoOp9t2UtlTV2wIzPGmDZhiSCpNwyYDCufB5+PM4amU1FTxxdfFwU7MmOMaROWCABGXwUlO2H7J0wYmEJ0hIcP1+cFOypjjGkTlggAhk6D6ERY+W+iI8KYODCVDzfm281qjDEhwRIBQEQ0DJ8O61+HimKmDE1nV1GF3azGGBMSLBHUG30V1FbCmpeZ4l5Gap3LjDGhwBJBvR6joNuJsPLfZCR14cSMBF5evtuah4wxnZ4lgnoiMOpK2LMC8tYxc0I/NuZ5WbR5b7AjM8aYgLJE4G/EZeCJgKznuWhUT9Lio3h80bZgR2WMMQFlicBfbCocdx6smkuU+Lh6Ql8Wbd7LhtzSYEdmjDEBY4mgsdFXwf69sOldrhzfl+gID08s+jrYURljTMBYImhs4JkQ1x1W/puusZFMH9ubV7N2k19qI5IaYzonSwSNhYXDqBmw+T3w5vKDSf2p9Sn/WrI92JEZY0xAWCJoyqirQOtg1Qv0S43l7OO78e/Pd7K/ujbYkRljTKuzRNCU1EHQ71RY8neo8nLNaQMoqajhpeXZwY7MGGNanSWC5px1J5QXwGcPkdm3KyN7J/Hkp19T57MOZsaYzsUSQXN6ZcKwS2Dxw4g3h2tO7c+Owv0sWGejkhpjOhdLBC056w7nXMFHf+DcYd3JSOrCE9bBzBjTyVgiaEnXfjDuWlj5POEF65g9qT/Lduxj+Y59wY7MGGNajSWCQzn15869ChbczuUn9SY1Loo7XltDbZ0v2JEZY0yrOKxEICLPHc6yTikmGU77JWz9gLhdH/O7C4exZncpT3+2PdiRGWNMqzjcGsEw/yciEgaMbf1w2qlx10BSX1hwO1OHpXHW8encv2AjOwv3BzsyY4w5Zi0mAhG5VUS8wAgRKXUnL5APvNomEbYH4VHOieO8Ncjqudx98YmEezz8ev5Xdr8CY0yH12IiUNU/qmo8cK+qJrhTvKqmqOqtbRRj+zDsEsjIhA9/T48uPm4+9zgWbd7LvJW7gx2ZMcYck8NtGnpDRGIBROQqEfmLiPQ91E4ikiQiL4nIBhFZLyITGq0XEXlIRLaIyGoRGXMUZWgbInDO78GbA5/cy1Xj+zKmTxJ3v7GOwrKqYEdnjDFH7XATwSPAfhEZCfwc2Ao8exj7PQi8o6pDgZHA+kbrzwMGu9O17uu0X30nOOMQffYQntws7vnOCMqqarn7jXXBjswYY47a4SaCWnUawy8C/qaqfwfiW9pBRBKB04AnAVS1WlWLG212EfCsOj4HkkSkxxGVoK196/cQmwav/pghKVFcP3kQ87P28PGmgmBHZowxR+VwE4FXRG4Fvge8KSIeIOIQ+/QHCoCnRWSliDxR37zkJwPY5fc82112EBG5VkSWiciygoIgf+F26QrT/gp5a+DTv/CjKQMZmBbLba98RVmVjU5qjOl4DjcRXA5UAbNVNRfoBdx7iH3CgTHAI6o6GigHbjmaIFX1MVXNVNXMtLS0ozlE6xo6FYZPh0/uJWrvev70nRHklFRw+6trgh2ZMcYcscNKBO6X//NAoohMAypV9VDnCLKBbFX9wn3+Ek5i8Lcb6O33vJe7rP07909O7eDVH5HZO4EbzhjMKyt2M2+lDVVtjOlYDrdn8WXAl8B04DLgCxG5tKV93OSxS0SOcxedCTQ+q/oaMNO9euhkoERVc46kAEETmwJT74OcLFj8EDecMYhx/ZL5zbw1bN9bHuzojDHmsB1u09CvgZNU9WpVnQmMA357GPvdADwvIquBUcD/icgcEZnjrn8L2AZsAR4H/t8RRR9swy6G4y+EhfcQXrSZv14xivAwDz+Zu5LqWhuLyBjTMRxuIvCoar7f88LD2VdVs9y2/RGqerGq7lPVR1X1UXe9quqPVHWgqg5X1WVHUYbgOv9+iIyBV39ERnwEf/rOCFZnl3DfexuDHZkxxhyWw00E74jIuyIyS0RmAW/i/Jo3celOE1H2UlhwO+ee2J2rTu7DY59ss0tKjTEdwqHGGhokIhNV9ZfAP4ER7rQEeKwN4usYhl8K4+fA53+HFc/xm/NPYEi3OH7+3ywKvNbr2BjTvh2qRvAAUAqgqq+o6k2qehMwz11n6p3zBxgwGd74GdE5S/nbd8fgrazlZy9m2b0LjDHt2qESQTdV/arxQndZv4BE1FGFhcOlT0NSb3jxKoZEFXP3RSfy6Za93PHaWhul1BjTbh0qESS1sK5LawbSKcQkw4wXobYK5s7gspHJzDl9IM9/sZMnFn0d7OiMMaZJh0oEy0TkmsYLReSHwPLAhNTBpQ2BS5+CvLUw/3puPmcwU4d35//eXs87azpGFwljTGgJP8T6G4F5InIlB774M4FI4NuBDKxDG3w2nH0XvPcbPGl/5i+X3UxOyefc+GIWcxO7MKp3SxUtY4xpW4e6MU2eqp4C/A7Y7k6/U9UJbs9h05wJP4ZRV8LH9xCd9QyPz8wkLT6KH/5rKbuK7BaXxpj243DHGvpIVR92pw8DHVSnIAIXPAhDzoU3f07qttd4etZJVNf6mP3MUkoqaoIdoTHGAIffocwcjbAImP4M9JsE865j0L5PefR7Y9leWM7sZ5ZSWmnJwBgTfJYIAi2iC8x4AXqMhP9ezSme9Tx0xWhWZxdz5eNfUFReHewIjTEhzhJBW4iKh6tehuT+8MIVnJe8h8e+l8nGPC9XPLaEfG9lsCM0xoQwSwRtJSYZvjcPYlLg399hSnIhz8w6iex9FVz26BJ2F1cEO0JjTIiyRNCWEnrCzFchLAqensop0Tt47gfjKCyv5rJHl9h9DIwxQWGJoK0l94fZb0N0AvzrAsbWrOSFa05mf3Ut0/+5hA25pcGO0BgTYiwRBEPyAJj9nvP4n8s5sWgBL143AY/A9EeWsHjL3mBHaIwJIZYIgiW+G3z/Teg9Dl7+AUO+fp5X/t9EuidGc/XTX/JqVse4dbMxpuOzRBBM0Ylw1SswdBq88ysylt/HS9dNYEyfrvx0bhaPLNxqo5YaYwLOEkGwRUTD9H/BmJmw6D4S37yGZ68aygUje/Kndzbw21fXUOezZGCMCZxDDTpn2kJYOFzwECQPhA/uIir3Kx689Bl6Jg3gnx9vI6+0ikeuHEN4mOVtY0zrs2+W9kIEJt0IV78O1eV4njqbW7st4/ZpJ7BgXR5PfWb3MzDGBIYlgvam30SY8yn0Hg+v/Zjv7/0z5w9N5C8LNrGz0EYtNca0PksE7VFcmtML+fRbkKwXeMD7c7p5Svn1/K/s5LExptVZImivPGEw5Va46iUiSnbwSuJfWbF5F/NW2mWlxpjWZYmgvRt0Flz2L5K9m/hP/MP88fVVNmKpMaZVWSLoCIZ8C7nob4ysyeKOuof5w+trgh2RMaYTsUTQUYz6Lpz1O6Z5ljBszZ/4ZGN+sCMyxnQSlgg6kok/pXbcHGaHv8Pal+5if3VtsCMyxnQClgg6EhHCz/0je/tdwPU1z/HeU3dRUmY3tTHGHBtLBB2Nx0PqVU+xJWE8F+c+SPG9I5j/2J2s3Z4T7MiMMR2UJYKOKDySQTe+zc6zHkViU7h4z1/p+XQm//vzdby9ZBXVtb5gR2iM6UCko3VQyszM1GXLlgU7jPZDlbItn7H3vfvoU7CQGg3j5YhpJEy9g6mj+uPxSLAjNMa0AyKyXFUzm1pnNYKOToS4wZPo96P58KOlFA24kO/Wzmfo/PO5+cEnWbzVbnJjjGmZJYJOxJM2mB5XP43vylfoGevjTyW/ZNXTN3LtU5+xMdcb7PCMMe1UQBOBiGwXka9EJEtEvtGeIyKTRaTEXZ8lIrcHMp5Q4Rl8JjE//RIddSXXh7/OzTuv5VcPPc0dr66htLIm2OEZY9qZtqgRTFHVUc21TQGL3PWjVPWuNognNEQnEn7x3+DKlxkQV8crkXcwYNnvuOS+N3hzdY4NXmeMaWBNQ53d4LPw/OhzPJnfZ2b4+8yr+zFfvvhHfvj0EnYV2bDWxpgAXzUkIl8D+wAF/qmqjzVaPxl4GcgG9gC/UNW1TRznWuBagD59+ozdsWNHwGLu1PLW4nvnVjxff8xWzeCPvu8xYvKlzBjXh7T4qGBHZ4wJoJauGgp0IshQ1d0ikg4sAG5Q1U/81icAPlUtE5GpwIOqOrilY9rlo8dIFTa+Re3btxFesp2P6kbyd98lxAw8hegADAsAABZ2SURBVG+P7sk5J3QnNsruYGpMZxO0RNAoiDuBMlW9r4VttgOZqtrsNY+WCFpJbRV88Sh1n9xPWFUJq2QoD1edz+KwTM46oQeXZfZm4qAURKwfgjGdQVD6EYhIrIjE188D5wBrGm3TXdxvGhEZ58ZTGKiYjJ/wKJj4U8JuWgfn/okRCWU8EXk/C2NuIXnTXGY/+SkX/2Mx76/LsxPLxnRyAasRiMgAYJ77NBz4j6r+QUTmAKjqoyLyY+B6oBaoAG5S1cUtHddqBAFSVwNr58PiByH3Kyojk/mvbwr/LDuNhB4DueGMQZw7rLv1VDamg2oXTUOtxRJBgKnCto/gy8fRTe+AwpKwsfyz4gz2pEzgusmDuWBkD6LCw4IdqTHmCFgiMEeneCcsfwZd8SxSXsAeT3eeqjqT97t8i0tPOYErx/ela2xksKM0xhwGSwTm2NRWw/rX0KVPIDuXUCFdmFtzGv+R8xg3JpPZk/ozMC0u2FEaY1pgicC0nj0r4fNH0DUvo746PvSN5fGa84gYMIkZ4/ty9gndiAy3forGtDeWCEzrK82BpU/gW/YUnooitkofnqk+g4+jzuC8k4ZwxUl96J8aG+wojTEuSwQmcKr3w5qX0KVPIjlZVEo082on8u/aM0noP5ZLxmRw7ondiY+OCHakxoQ0SwSmbexeDkufQte8jNRWsMEziDeqRvOpjKHX8eO5ZGwvTh2cRkSYNR0Z09YsEZi2VbEPVs1FV7+I7FkJQD5d+aB2FEsjT6L/uGlcf/Zwwi0hGNNmLBGY4CnLh80L8G16F9+WDwivKaNYY1mQcAlnXn07yanpwY7QmJBgicC0D3U1sGMxu999gIy8DykjhrJRP6D7OTdBTHKwozOmU7N7Fpv2ISwCBpxOxvXz2HLJO3zpGUn3rIepuX8YLLgDincFO0JjQpLVCEzQ7Cuv5s/PvsKEPc8wLexzPCj0GAlDL4Djp0HaULDRT41pFdY0ZNqtOp/y1wWbeHXhZ1wUsYxLYrIYUOnemyh5ABx/AYy6EtKOC26gxnRwlghMu7d8RxEvLd/Nu2tzCSvP4/zIlVwWt4qhFVl4tAb6ToTM2U5iCLe7qRlzpCwRmA6jts7HF18X8eZXOby7JhfKC5jZ5TNmRi6ka1U2xKQ4NYQxMyG1xZvZGWP8WCIwHVJtnY9Pt+zl+S928uH6HE6Rtfw06VPGVCzGo3WQ1AcGTIb+pztTXFqwQzam3bJEYDq8PcUVzP1yJy8s3QXeXKbHrOSMqPWcWL2K6LoyAHzpw/AMmAz9T4O+p0B0QlBjNqY9sURgOo2aOh/vrc3j/fV5bMz1si2/lCG+rUzyrGFi2BoyPZuJohofYRQlDaOq9yRihp5B0sDxiCUGE8IsEZhOq7bOx/bCcjbketmY62VHXhHRucvp713GONYwUrYSLj4A9kX2RLsNI6nfKDzdT4TuwyFlYJBLYEzbsERgQo7Pp+wurmBHTi5lmxZRtn0FMfs2MJhdDPDsIQzn776u23DCxnwPhk+33s2mU7NEYAxQVlXLok0FLFy7k50bVzKkeh3TwxdxomyjzhOBDplK+NiZMHAKeOyezKZzsURgTCN1PmXp9iLe+iqHzas/5+yqBXw77DO6ipfKqBSqep1CzODJRAw8FVKHWA9n0+FZIjCmBXU+5cuvi3hn1XYq17zJhJoljPdsoIcUAVAalsSexLHU9RpHxnEnkdR/tDUjmQ7HEoExh6nOp2zK87Ilz0vBzg1EZi+m+75lnFD9FT2lsGG74oh0KpNPILbvKOJHXAAZY63WYNo1SwTGHKOaOh8bt25lx9ovKN+ZRcy+9Qz2bWeg7CFcfNSlHU/YmJkw4nKITQl2uMZ8gyUCY1pZnU/ZkFvK+ys3k7fkBa6K/JgTfJshLBKGnu8Mg5Ex1pqQTLvRUiIIb+tgjOkMwjzCsJ6JDOuZyfITB3DNC+eT6N3EPRlZDN/2DrJ2nrNhTAqkHueMi5R2nHPiOXkAJPWFMPv3M+2D1QiMaQUlFTXc9spXvPlVDpMHJvDAeC9JZdtg70bYuxkKNkJF0YEdPBHQtZ/ToS1lEGSMgUFn27AYJmCsaciYNqCqzF26i9+9vpboiDAmDUplZK8kRvRK5MSMRGJrS2DvJijaCoVb3Gmb87y20mlW6n+607R03FSI7xbsIplOxBKBMW1oc56XB97fzMqd+9hTUgmAR2BQehwn9kxkYHocA9PiGJQeS5/kWCI9Cru+hA1vONO+7YBAr0zodZLTrJQ6xGliik21q5PMUbFEYEyQFHir+Gp3Mat2lbA6u5h1OaXklVY1rA/zCH2SY+jVtQvp8dGkx0cyxJPNCSWLyMhfSGzxJqS24sABo5Occw09R0PPMc4J6eQB4LHbj5uWWSIwph0pq6plW0EZWwvK2FZQzpb8MvYUV5DvraLAW0Wt78D/pOBjWKyXSV2LGBVdwCDPHrpXfU1s0VqkZr+zUVQiZIyG7iOc2kOye94hLt1qD6aBXTVkTDsSFxXOiF5JjOiV9I11Pp+yb381+d4qcksr2ZpfxsZcL4vzvDyzw0tljTOSanSYj3PSSjgjIZsRspUM73oidzyK1FUfOFhkPKQMgG7Doe8E6DPBqT1YcjCNWI3AmA6izqfsKtrPupxSVmUXs2pXMauzS9hfXQdAQpSHU1L3My5hHydE5tOXXFKrdhCRmwWVxc5B4rpBn5Oh98mQmAGRcRCVAFHxztSlK0TGBLGUJlCsaciYTqrOp2zJL2PVrmK+2l3CxjznvgwlFTUN28RECCOj8jg5fBOjWc+wmrWk1OU3fUDxOHd4O+FiOP5C6yXdiQQtEYjIdsAL1AG1jYMQEQEeBKYC+4FZqrqipWNaIjCmZapKgbeq4WY9eaWVlFbWUFpRS0lFDaWVNYSV51HjLSBGK4iXCuKooEeXGkbGFnOOfEFkydcgYU5SGPZtGHw2xHW3k9IdWLATQaaq7m1m/VTgBpxEMB54UFXHt3RMSwTGtI6q2jp2Fu5na0E5X+8tZ1tBGR9uyKe8uoYHp0Rwjm8Jsm4eFG1zdgiLhIQMSOwFib2dx5RB0G2Yc3lreGRwC2Ra1J5PFl8EPKtONvpcRJJEpIeq5gQ5LmM6vajwMAZ3i2dwt/iGZfneSm6cm8V17xVyyegLufvaW4ndt87p51CyC0qynenrj8GbA+qcvMYT4VzW2m2YM3Xt7ySKpD7OMBt2grpdC3QiUOA9EVHgn6r6WKP1GcAuv+fZ7rKDEoGIXAtcC9CnT5/ARWtMiEuPj+a5H4zn4Q838+AHm1mVXczfrxzD0HEjv7lxXY3TOzpvLeStgdw18PUiWP3iwduFR7tJoa9zn+geI6HnKCdZWIJoFwLdNJShqrtFJB1YANygqp/4rX8DuEdVP3WffwD8SlWbbfuxpiFj2sbirXv56dwsSitquO60AfRM6kJsVDhx0eHERYUTGxlOn5QY4qIa/Z6s2AfFO52aQ/GuAzWJoq2QvwF87onsqEToMcJJDskDnHGXkgc6ScNuFdrqgtY0pKq73cd8EZkHjAM+8dtkN9Db73kvd5kxJshOGZjKWz85lZv+m8VDH25pchuPwNDuCZzUrytj+yWT2bcrPZO6Opeh9miiFlFbBfnrIGcV7MmCnCxY9jT4954Oi4Lk/k6NofFjUh87FxEAAasRiEgs4FFVrzu/ALhLVd/x2+Z84MccOFn8kKqOa+m4ViMwpu2VVtZQXlVLWWUt3qpayqtq8VbWsiHXy/IdRazcWdzQn6F7QjRdIsOorvVRU+ejus5HTa0Pjwh9U2MYlBbHoPQDU1pcFLXFe/AVbkELt+Ip2kp48dfElu8kvHTngR7U4FzemtQHUgY7vahTBjrzKYMgvrvVJFoQrBpBN2Cec4Uo4cB/VPUdEZkDoKqPAm/hJIEtOJePfj+A8RhjjlJCdAQJ0RGQePDyqcN7AFBb52N9jpdlO4pYnV1CrU+JCBMiwzxEhnuICPNQW+dj295yvvy6iPlZe5p5pV7udDoA6XGRnN7XxyldvQyPKaIPuc6lrYWbYcdn4J8kPBFOJ7nE3k6ySOztdJKr8kJVqTt5nSk23Tmp3f1E6HaiMxxHCLMOZcaYNldeVcvWgjK25JdRVF5NVEQYUeEedwojMlzYWbiflbuKydpVzI5C5wtfBLrFR5PRtQu9EqM4LraMIeG59NYcutbkEV+ZQ1T5bjwl2c5VTbjfb5HxB3pPR8Y667x+16TUJ4YeI5wxm3qMdM5XdKJ+E9az2BjToRWVVzcMqbFr335276sgu3g/OcWVBw3SVy+xSwTdY4XjUyP51phBTDm+O9ERjZqNygshf61ztVPeWsj7CvLXQ/14TZFxTm2h2zDnBHZCT3fKgPgeHW4oDksExphOqc6n5Hsr2VNcyd6yKmfyVjfML9uxjwJvFfHR4Zw/vAcXj85gXL9kPJ5mLlutrXbuKpezCnJWO48F66Gy5JvbxqQ4VzslD3QfBziD/KUMgujEb24fZJYIjDEhqbbOx+KthczP2s07a3LZX11Hz8RoMvsl0yMpmoykLvRI7EJPdz6xSwTSVN+G6nIozQHvHijdA6W7nUtki7Y5d5krzT54+7hu7s2E3JsKpQyGpN5ObSIqrm0K34glAmNMyNtfXcuCdXm8vmoPm/LKyC2ppLrOd9A2cVHh9E6OoXfXLvRJjqFPyoGbBqXGRZESF0lEWBPnDWoqnDvLFbq3Id272bkt6d5NB0Z+rReddGCIjoSezonq2DRnqp+P7+6cy2hFlgiMMaYRn0/ZW15FTnEle4or2F1cQfa+CnYV7Wdn0X527dvfcP8Hf8mxkaTFRZEWH0VqXCSpDfPO48D0OHomRjs1C1XYX+gkhpJsp+ZQP0xHyW6nZlFR1HSADQkjwx3jKQP6ToI+LQ7H1qz2PNaQMcYEhccj7u1BoxnZ+5s3CVJVCsqq2L2vggJvFQVlzh3kGqayKnbsLGevt5qKmrqD9k2JjWR4r0RGZCQyvFcSQ7uPxBc3gtLkWnckWGcUWJ/CkLQohsZXE1tTBGUFUJ7vXNFUnyhKdsOuL5we26f+4qgTQUssERhjTBNEDiSKlqgq5dV17PVWke+tYmOel6+ynSucFm3eS10TVzU1pW9KDMd378rQHn0YkBZHeoZTw0iLjyI+KtzpWOerO/SBjoIlAmOMOQYiQlyUM/5Sv9RYxvVPBvoCUFFdx7qcEjbmlhEV7iGhSwQJ0eHOY5cIfD5lY66X9TmlbHAf312XS+MW++gID2nxUcw8uR/XnJbQ6mWwRGCMMQHSJTKMsX2TGds3udlteifHcNYJ3Rqe76+ubWiOynebofK9lRR4q0hPiApInJYIjDGmHYmJDP/GfSICrfP0nzbGGHNULBEYY0yIs0RgjDEhzhKBMcaEOEsExhgT4iwRGGNMiLNEYIwxIc4SgTHGhLgON/qoiBQAOw5j01Rgb4DDCbZQKCOERjmtjJ1Hey1nX1VNa2pFh0sEh0tEljU35GpnEQplhNAop5Wx8+iI5bSmIWOMCXGWCIwxJsR15kTwWLADaAOhUEYIjXJaGTuPDlfOTnuOwBhjzOHpzDUCY4wxh8ESgTHGhLhOlwhE5FwR2SgiW0TklmDHc6REZLuIfCUiWSKyzF2WLCILRGSz+9jVXS4i8pBb1tUiMsbvOFe7228WkauDVR6/eJ4SkXwRWeO3rNXKJSJj3fdti7uvtG0Jmy3jnSKy2/08s0Rkqt+6W914N4rIt/yWN/k3LCL9ReQLd/mLIhLZdqVriKG3iHwkIutEZK2I/NRd3tk+y+bK2ak+zwaq2mkmIAzYCgwAIoFVwAnBjusIy7AdSG207M/ALe78LcCf3PmpwNuAACcDX7jLk4Ft7mNXd75rkMt1GjAGWBOIcgFfutuKu+957aSMdwK/aGLbE9y/zyigv/t3G9bS3zDwX+AKd/5R4PoglLEHMMadjwc2uWXpbJ9lc+XsVJ9n/dTZagTjgC2quk1Vq4G5wEVBjqk1XAT8y53/F3Cx3/Jn1fE5kCQiPYBvAQtUtUhV9wELgHPbOmh/qvoJUNRocauUy12XoKqfq/Nf9azfsdpMM2VszkXAXFWtUtWvgS04f79N/g27v4rPAF5y9/d/v9qMquao6gp33gusBzLofJ9lc+VsTof8POt1tkSQAezye55Nyx9ee6TAeyKyXESudZd1U9Ucdz4XqL/TdXPl7SjvQ2uVK8Odb7y8vfix2yzyVH2TCUdexhSgWFVrGy0PGhHpB4wGvqATf5aNygmd8PPsbImgM5ikqmOA84Afichp/ivdX0md7prfzlou4BFgIDAKyAHuD244rUNE4oCXgRtVtdR/XWf6LJsoZ6f8PDtbItgN9PZ73std1mGo6m73MR+Yh1O1zHOrzLiP+e7mzZW3o7wPrVWu3e584+VBp6p5qlqnqj7gcZzPE468jIU4zSrhjZa3ORGJwPlyfF5VX3EXd7rPsqlydsbPEzpfIlgKDHbPxkcCVwCvBTmmwyYisSISXz8PnAOswSlD/VUVVwOvuvOvATPdKzNOBkrc6vm7wDki0tWtup7jLmtvWqVc7rpSETnZbXud6XesoKr/cnR9G+fzBKeMV4hIlIj0BwbjnCRt8m/Y/ZX9EXCpu7//+9Vm3Pf3SWC9qv7Fb1Wn+iybK2dn+zwbBOssdaAmnKsUNuGcqf91sOM5wtgH4FxVsApYWx8/TnviB8Bm4H0g2V0uwN/dsn4FZPodazbOCastwPfbQdlewKlK1+C0h/6gNcsFZOL8U24F/obba74dlPE5twyrcb4sevht/2s33o34XRnT3N+w+/fxpVv2/wFRQSjjJJxmn9VAljtN7YSfZXPl7FSfZ/1kQ0wYY0yI62xNQ8YYY46QJQJjjAlxlgiMMSbEWSIwxpgQZ4nAGGNCnCUCE3JEpMx97Cci323lY9/W6Pni1jy+MYFgicCEsn7AESUCv56gzTkoEajqKUcYkzFtzhKBCWX3AKe648r/TETCROReEVnqDip2HYCITBaRRSLyGrDOXTbfHRhwbf3ggCJyD9DFPd7z7rL62oe4x14jzlj7l/sde6GIvCQiG0TkebdXKyJyjzjj4a8Wkfva/N0xIeNQv26M6cxuwRlbfhqA+4VeoqoniUgU8JmIvOduOwY4UZ0hhgFmq2qRiHQBlorIy6p6i4j8WFVHNfFal+AMVDYSSHX3+cRdNxoYBuwBPgMmish6nCEMhqqqikhSq5feGJfVCIw54ByccXGycIYcTsEZMwbgS78kAPATEVkFfI4zqNhgWjYJeEGdAcvygI+Bk/yOna3OQGZZOE1WJUAl8KSIXALsP+bSGdMMSwTGHCDADao6yp36q2p9jaC8YSORycBZwARVHQmsBKKP4XWr/ObrgHB1xqkfh3PjkmnAO8dwfGNaZInAhDIvzm0I670LXO8OP4yIDHFHgW0sEdinqvtFZCjObRXr1dTv38gi4HL3PEQazm0tv2wuMHcc/ERVfQv4GU6TkjEBYecITChbDdS5TTzPAA/iNMuscE/YFtD07QPfAea47fgbcZqH6j0GrBaRFap6pd/yecAEnJFlFbhZVXPdRNKUeOBVEYnGqancdHRFNObQbPRRY4wJcdY0ZIwxIc4SgTHGhDhLBMYYE+IsERhjTIizRGCMMSHOEoExxoQ4SwTGGBPi/j95qHGHrwMMQgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FG-DQTNVdy9r"
      },
      "source": [
        "Let’s quickly save our trained model:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rFpmgNyCdy9w"
      },
      "source": [
        "PATH = '/content/drive/My Drive/Transformer3.pth'\n",
        "torch.save(model3.state_dict(), PATH)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WoUInW7Rdy9y",
        "outputId": "4b7c7a6c-5745-45c5-93f9-2a710f37f92e"
      },
      "source": [
        "print(train_losses3)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[7.12672170829773, 6.46189419555664, 6.236063598632812, 6.0882855920791625, 5.984243896484375, 5.893051532745361, 5.8159049062728885, 5.752507901191711, 5.693956283569336, 5.589580288887024, 5.561838900566101, 5.5255132074356075, 5.484324089050293, 5.460690399169922, 5.428388655662537, 5.398866388320923, 5.372425700187683, 5.348479837417602, 5.296341809272766, 5.274308337211609, 5.247021976470947, 5.2354551630020145, 5.225406174659729, 5.205230780601502, 5.182973200798035, 5.172115510940552, 5.155914067268371, 5.107554347991943, 5.0938058843612675, 5.083502143859863, 5.07703448677063, 5.06543288230896, 5.051340876579284, 5.039793697357178, 5.0355708208084105, 5.025488792419433, 4.975865616798401, 4.972390535354614, 4.956974133491516, 4.958637884140015, 4.9479644947052, 4.950071088790893, 4.935765158653259, 4.92771598148346, 4.921769055366516, 4.897920921325683, 4.89029215335846, 4.881584662437439, 4.875364864349366, 4.869326411247253, 4.855814215660096, 4.85994566822052, 4.847824262619018, 4.841476984024048]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cpfUsf_cdy9z",
        "outputId": "5939d1af-dd87-4ae5-fd59-7f4eb3f70b47"
      },
      "source": [
        "print(validation_losses3)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[6.595450358604317, 6.319155782905977, 6.145209693196994, 6.025102307547384, 5.924179872470115, 5.843560733901921, 5.772663861068327, 5.712082989180266, 5.661628710689829, 5.588227617206858, 5.558168792902534, 5.513683267493747, 5.475574759405051, 5.445043104798047, 5.417688414232055, 5.388465998777702, 5.361304210193121, 5.341884521406088, 5.309326466339738, 5.288369896696575, 5.261506803889773, 5.244026640457894, 5.2267997478371235, 5.214895077605746, 5.189556590656736, 5.173087774817623, 5.160829005846336, 5.13502259930568, 5.124514014863256, 5.114749619320257, 5.09394990419274, 5.083499680704145, 5.069631059667957, 5.059901949185044, 5.044191746569392, 5.040572052571311, 5.018282630550328, 5.01050619936701, 4.998459825764841, 4.990631580352783, 4.977491056741173, 4.967902800040458, 4.958835430109679, 4.948803663253784, 4.940747006615596, 4.944426836362526, 4.932391153342688, 4.919563938432665, 4.912429244660619, 4.902979107045415, 4.892674973651545, 4.88501096572449, 4.879061398221485, 4.871543209944198]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oUiBPmMYHkYB"
      },
      "source": [
        "train_losses3 = [7.12672170829773, 6.46189419555664, 6.236063598632812, 6.0882855920791625, 5.984243896484375, 5.893051532745361, 5.8159049062728885, 5.752507901191711, 5.693956283569336, 5.589580288887024, 5.561838900566101, 5.5255132074356075, 5.484324089050293, 5.460690399169922, 5.428388655662537, 5.398866388320923, 5.372425700187683, 5.348479837417602, 5.296341809272766, 5.274308337211609, 5.247021976470947, 5.2354551630020145, 5.225406174659729, 5.205230780601502, 5.182973200798035, 5.172115510940552, 5.155914067268371, 5.107554347991943, 5.0938058843612675, 5.083502143859863, 5.07703448677063, 5.06543288230896, 5.051340876579284, 5.039793697357178, 5.0355708208084105, 5.025488792419433, 4.975865616798401, 4.972390535354614, 4.956974133491516, 4.958637884140015, 4.9479644947052, 4.950071088790893, 4.935765158653259, 4.92771598148346, 4.921769055366516]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LKpsRppREL0E"
      },
      "source": [
        "validation_losses3 = [6.595450358604317, 6.319155782905977, 6.145209693196994, 6.025102307547384, 5.924179872470115, 5.843560733901921, 5.772663861068327, 5.712082989180266, 5.661628710689829, 5.588227617206858, 5.558168792902534, 5.513683267493747, 5.475574759405051, 5.445043104798047, 5.417688414232055, 5.388465998777702, 5.361304210193121, 5.341884521406088, 5.309326466339738, 5.288369896696575, 5.261506803889773, 5.244026640457894, 5.2267997478371235, 5.214895077605746, 5.189556590656736, 5.173087774817623, 5.160829005846336, 5.13502259930568, 5.124514014863256, 5.114749619320257, 5.09394990419274, 5.083499680704145, 5.069631059667957, 5.059901949185044, 5.044191746569392, 5.040572052571311, 5.018282630550328, 5.01050619936701, 4.998459825764841, 4.990631580352783, 4.977491056741173, 4.967902800040458, 4.958835430109679, 4.948803663253784, 4.940747006615596]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7KCjLjdQDJ96"
      },
      "source": [
        "## Test "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 267,
          "referenced_widgets": [
            "72627ae0bda64d73971adcb6a8cd7b5d",
            "544f94b1c85743ed91814f6c169fea7b",
            "a011ce0d39e349b0bed99d65bd900830",
            "a6f369c947474f588efafda627c23a9c",
            "9b24eee8c1974db0bfc8abff3b444beb",
            "74b3b5dec01e4caaa608a1a0f4b91255",
            "76c11f27149047228c7a5de4a25b4b39",
            "709f58e689164a51b73116d581c5233b"
          ]
        },
        "id": "JBlGVh77DJ-D",
        "outputId": "1f65ed0e-d606-4814-ad85-c8fb5e6d9423"
      },
      "source": [
        "torch.manual_seed(2045)\n",
        "\n",
        "pad_idx = 0\n",
        "\n",
        "# Parameters\n",
        "params = {'batch_size': 1,\n",
        "          'shuffle': False,\n",
        "          'num_workers':2,\n",
        "          'collate_fn' : BatchPadCollate(pad_idx=pad_idx)\n",
        "          }\n",
        "\n",
        "# Training and validation Data Generators\n",
        "test_set0 = AFEC_Dataset(en_test_dic,fa_test_dic0,partition['test'],en_vocab_dic, fa_vocab_dic)\n",
        "test_generator0 = torch.utils.data.DataLoader(test_set0, **params)\n",
        "test_set1 = AFEC_Dataset(en_test_dic,fa_test_dic0,partition['test'],en_vocab_dic, fa_vocab_dic)\n",
        "test_generator1 = torch.utils.data.DataLoader(test_set1, **params)\n",
        "test_set2 = AFEC_Dataset(en_test_dic,fa_test_dic0,partition['test'],en_vocab_dic, fa_vocab_dic)\n",
        "test_generator2 = torch.utils.data.DataLoader(test_set2, **params)\n",
        "test_set3 = AFEC_Dataset(en_test_dic,fa_test_dic0,partition['test'],en_vocab_dic, fa_vocab_dic)\n",
        "test_generator3 = torch.utils.data.DataLoader(test_set3, **params)\n",
        "\n",
        "\n",
        "# Hyperparameters\n",
        "embed_dim = 256\n",
        "num_heads = 8\n",
        "dim_key = 64\n",
        "dim_value = 64\n",
        "dim_inner = 1024\n",
        "num_layers = 3\n",
        "src_vocab_size = len(en_vocab_dic.get_str_2_idx())\n",
        "trg_vocab_size = len(fa_vocab_dic.get_str_2_idx())\n",
        "learning_rate = 1e-4\n",
        "\n",
        "kwargs = {'p_dropout': 0.1,\n",
        "          'kdim': dim_key,\n",
        "          'vdim':dim_value,\n",
        "          }\n",
        "# Model\n",
        "# PATH = '/content/drive/My Drive/Transformer3.pth'\n",
        "# model3 = Transformer2(num_layers, embed_dim, dim_inner, num_heads, src_vocab_size,trg_vocab_size, device,**kwargs).to(device)\n",
        "# model3.load_state_dict(torch.load(PATH))\n",
        "\n",
        "test_dataiter0 = iter(test_generator0)\n",
        "test_dataiter1 = iter(test_generator1)\n",
        "test_dataiter2 = iter(test_generator2)\n",
        "test_dataiter3 = iter(test_generator3)\n",
        "\n",
        "list_of_references = []\n",
        "hypothesises = []\n",
        "bleu_scores = []\n",
        "nist_scores = []\n",
        "\n",
        "# Loop over batches\n",
        "for num_batch in tqdm(range(len(test_generator0))):\n",
        "    # Test \n",
        "    with torch.set_grad_enabled(False):\n",
        "        # Transfer to GPU\n",
        "        batch_src0, batch_trg0 = test_dataiter0.next() \n",
        "        batch_src0, batch_trg0 = batch_src0.to(device), batch_trg0.to(device)\n",
        "        batch_src1, batch_trg1 = test_dataiter1.next() \n",
        "        batch_src1, batch_trg1 = batch_src1.to(device), batch_trg1.to(device)\n",
        "        batch_src2, batch_trg2 = test_dataiter2.next() \n",
        "        batch_src2, batch_trg2 = batch_src2.to(device), batch_trg2.to(device)\n",
        "        batch_src3, batch_trg3 = test_dataiter3.next() \n",
        "        batch_src3, batch_trg3 = batch_src3.to(device), batch_trg3.to(device)\n",
        "\n",
        "        # Model computations\n",
        "        # forward + backward + optimize\n",
        "        batch_outputs1 = model3(batch_src0, batch_trg0[: , :-1])\n",
        "        batch_outputs = model3.translate_sentence(batch_src0, batch_trg0.shape[1] + 5, 1, 2)\n",
        "        batch_outputs = torch.Tensor(batch_outputs)\n",
        "\n",
        "        references = [fa_vocab_dic.IndexDecoding(batch_trg0[0,:]),fa_vocab_dic.IndexDecoding(batch_trg1[0,:]),\n",
        "                      fa_vocab_dic.IndexDecoding(batch_trg2[0,:]),fa_vocab_dic.IndexDecoding(batch_trg3[0,:])]\n",
        "        list_of_references.append(references)\n",
        "        \n",
        "        hypothesises.append(fa_vocab_dic.IndexDecoding(batch_outputs))\n",
        "        bleu_scores.append(sentence_bleu(references,fa_vocab_dic.IndexDecoding(batch_outputs)))\n",
        "        nist_scores.append(sentence_nist(references,fa_vocab_dic.IndexDecoding(batch_outputs)))\n",
        "        \n",
        "        # print(batch_src0.shape, batch_trg0.shape, len(batch_outputs))\n",
        "        # print(en_vocab_dic.IndexDecoding(batch_src0[0,:])[1:-1])\n",
        "        # print(fa_vocab_dic.IndexDecoding(batch_trg0[0,:])[1:-1])\n",
        "        # print(fa_vocab_dic.IndexDecoding(batch_outputs)[1:-1])\n",
        "        # if num_batch == 5:\n",
        "        #   break\n",
        "\n",
        "bleu_score = corpus_bleu(list_of_references, hypothesises) \n",
        "nist_score = corpus_nist(list_of_references, hypothesises, n = 5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "72627ae0bda64d73971adcb6a8cd7b5d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=250.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/nltk/translate/bleu_score.py:490: UserWarning: \n",
            "Corpus/Sentence contains 0 counts of 2-gram overlaps.\n",
            "BLEU scores might be undesirable; use SmoothingFunction().\n",
            "  warnings.warn(_msg)\n",
            "/usr/local/lib/python3.7/dist-packages/nltk/translate/bleu_score.py:490: UserWarning: \n",
            "Corpus/Sentence contains 0 counts of 4-gram overlaps.\n",
            "BLEU scores might be undesirable; use SmoothingFunction().\n",
            "  warnings.warn(_msg)\n",
            "/usr/local/lib/python3.7/dist-packages/nltk/translate/bleu_score.py:490: UserWarning: \n",
            "Corpus/Sentence contains 0 counts of 3-gram overlaps.\n",
            "BLEU scores might be undesirable; use SmoothingFunction().\n",
            "  warnings.warn(_msg)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SXyJ1i0qDJ-G",
        "outputId": "9cf6f57b-20ba-4ba0-e378-925cc1293c4b"
      },
      "source": [
        "print(bleu_score,nist_score)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.044645439571308496 0.001183667412512276\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0PLJui47DJ-H",
        "outputId": "fb131de9-af82-45c4-d4c7-3ed612c348c9"
      },
      "source": [
        "print(np.mean(bleu_scores),np.mean(nist_scores))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.5397333763223582 0.013743155283581648\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "68agszJbSFO7",
        "outputId": "d1d58d04-0609-41f4-a950-5119dad92e35"
      },
      "source": [
        "print(fa_vocab_dic.IndexDecoding(torch.max(batch_outputs1, dim = 2)[1][0,:]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['بله', 'ما', 'ساعت', 'عصر', 'امروز', 'که', 'ما', 'چیز', 'درباره', 'کنیم', 'ما']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 906,
          "referenced_widgets": [
            "564cfb8a5c7a400891e8d649537840ac",
            "88684275bc9d431eb07ca467a4c6445f",
            "101e4c20a0fd49e2aa66b139bb20f356",
            "448105e8446e488db15da2b775916587",
            "b317bad0279b48539bc4d7b1e6245442",
            "a1c302e6ed854b22bfc6aaf5db85e5f5",
            "bf912fc3357245c087ef9f526aa08ca8",
            "bd34281e9a5447b69a693127817b77a6"
          ]
        },
        "id": "33t5ng2YDJ-J",
        "outputId": "be8baf3d-5d36-405d-be20-505e91325b16"
      },
      "source": [
        "torch.manual_seed(2045)\n",
        "\n",
        "pad_idx = 0\n",
        "\n",
        "# Parameters\n",
        "params = {'batch_size': 1,\n",
        "          'shuffle': True,\n",
        "          'num_workers':2,\n",
        "          'collate_fn' : BatchPadCollate(pad_idx=pad_idx)\n",
        "          }\n",
        "\n",
        "# Training and validation Data Generators\n",
        "test_set = AFEC_Dataset(en_test_dic,fa_test_dic0,partition['test'],en_vocab_dic, fa_vocab_dic)\n",
        "test_generator = torch.utils.data.DataLoader(test_set, **params)\n",
        "\n",
        "# Hyperparameters\n",
        "embed_dim = 256\n",
        "num_heads = 8\n",
        "dim_key = 64\n",
        "dim_value = 64\n",
        "dim_inner = 1024\n",
        "num_layers = 3\n",
        "src_vocab_size = len(en_vocab_dic.get_str_2_idx())\n",
        "trg_vocab_size = len(fa_vocab_dic.get_str_2_idx())\n",
        "learning_rate = 1e-4\n",
        "num_iterations = 10\n",
        "\n",
        "kwargs = {'p_dropout': 0.1,\n",
        "          'kdim': dim_key,\n",
        "          'vdim':dim_value,\n",
        "          }\n",
        "# Model\n",
        "# PATH = '/content/drive/My Drive/Transformer1.pth'\n",
        "# model3 = Transformer2(num_layers, embed_dim, dim_inner, num_heads, src_vocab_size,trg_vocab_size, device,**kwargs).to(device)\n",
        "# model3.load_state_dict(torch.load(PATH))\n",
        "\n",
        "test_dataiter = iter(test_generator)\n",
        "\n",
        "# Loop over epochs\n",
        "for itr in tqdm(range(num_iterations)):\n",
        "    \n",
        "    # Test\n",
        "    batch_src, batch_trg = test_dataiter.next() \n",
        "    \n",
        "    with torch.set_grad_enabled(False):\n",
        "\n",
        "\n",
        "        # Model computations\n",
        "        # forward + backward + optimize\n",
        "        \n",
        "        batch_outputs = model3.translate_sentence(batch_src, batch_trg.shape[1] + 5, 1, 2)\n",
        "        batch_outputs = torch.Tensor(batch_outputs)\n",
        "\n",
        "        print(\"Sentence {}:\".format(itr + 1))\n",
        "        print(\" \".join(en_vocab_dic.IndexDecoding(batch_src[0,:])).replace(\"<sos>\", \"\").replace(\"<eos>\", \".\"))\n",
        "        print(\" \".join(fa_vocab_dic.IndexDecoding(batch_trg[0,:])).replace(\"<sos>\", \"\").replace(\"<eos>\", \"\").replace(\"<unk>\",\"ن.ش\"))\n",
        "        print(\" \".join(fa_vocab_dic.IndexDecoding(batch_outputs)).replace(\"<sos>\", \"\").replace(\"<eos>\", \"\").replace(\"<unk>\",\"ن.ش\"))\n",
        "        print(\"-\"*50)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "564cfb8a5c7a400891e8d649537840ac",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Sentence 1:\n",
            " once again shall we stay at the hotel <unk> .\n",
            " یکبار دیگر آیا قرار است در هتل ن.ش بمانیم \n",
            " ما را در ساعت ما آه آه آه آه آه ما میتوانیم ما میتوانیم میتوانیم ساعت\n",
            "--------------------------------------------------\n",
            "Sentence 2:\n",
            " fine let us go for a little walk in hamburg .\n",
            " خوب است اجازه دهید برای یک کوتاه در هامبورگ برویم \n",
            " شما شما ن.ش شما شما شما شما شما را شما شما شما شما شما را \n",
            "--------------------------------------------------\n",
            "Sentence 3:\n",
            " yes i have already booked the hotel .\n",
            " بله من در حال حاضر هتل را رزرو \n",
            " من من من را من و بیست من بیست و بیست و من و بیست\n",
            "--------------------------------------------------\n",
            "Sentence 4:\n",
            " in the evening we should watch the newest movie with arnold schwarzenegger .\n",
            " عصر باید جدیدترین فیلم آرنولد ن.ش را تماشا کنیم \n",
            " و ما را ن.ش \n",
            "--------------------------------------------------\n",
            "Sentence 5:\n",
            " where should we meet .\n",
            " کجا باید ملاقات کنیم \n",
            " ما ما میتوانیم ما را خواهیم ما خواهیم کنیم ما ما\n",
            "--------------------------------------------------\n",
            "Sentence 6:\n",
            " hello i would prefer the intercontinental .\n",
            " سلام من ن.ش را ترجیح میدادم \n",
            " \n",
            "--------------------------------------------------\n",
            "Sentence 7:\n",
            " i have asked when we will arrive in hanover .\n",
            " من چه وقت به هانوور خواهیم رسید \n",
            " من ما میتوانیم ما ما خواهیم خواهیم خواهیم خواهیم ما خواهیم خواهیم ما در\n",
            "--------------------------------------------------\n",
            "Sentence 8:\n",
            " fine let us meet at the station at nine oclock on thursday morning .\n",
            " خوب است اجازه دهید ساعت نه صبح ن.ش در ایستگاه ملاقات کنیم \n",
            " ساعت بیست بیست \n",
            "--------------------------------------------------\n",
            "Sentence 9:\n",
            " yes i have got a timetable when would you like to go .\n",
            " بله من یک جدول زمانی دارم چه وقت دوست داری بروی \n",
            " شما شما را شما ساعت شما شما شما را شما شما را شما را دارم \n",
            "--------------------------------------------------\n",
            "Sentence 10:\n",
            " i agree everything okay .\n",
            " موافقم همه چیز مرتب است \n",
            " من را من \n",
            "--------------------------------------------------\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i3KK-OOnRnBJ"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}