Transformer_for_Covid19_Tweet_classification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Transformer for Covid19 Tweet classification.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyPHplPTEi05rVsQ3rcKLxYl",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "d1fe038b94344df7a719b2471da9849b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5dd68006717f4f7eaa589f80ac4f042f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3c3e8383f81847d291a11ea06346784b",
              "IPY_MODEL_2cbaa5a166ba4701a311b042e3a75dfe"
            ]
          }
        },
        "5dd68006717f4f7eaa589f80ac4f042f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3c3e8383f81847d291a11ea06346784b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ef80ce6ca8064944aec4904394fcdb0a",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1a7c824f16744a80817727a94f915698"
          }
        },
        "2cbaa5a166ba4701a311b042e3a75dfe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_bdeec54153854382ae33e9b42378c60d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:02&lt;00:00, 111kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5a97e0a4bde24777a3c40392b5913b2e"
          }
        },
        "ef80ce6ca8064944aec4904394fcdb0a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1a7c824f16744a80817727a94f915698": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bdeec54153854382ae33e9b42378c60d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5a97e0a4bde24777a3c40392b5913b2e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b7ab52517f744351a01bf50ac296ab0d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_dcb253b6935e4ed0866c241cb44722a1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_36246122c68d4a20918f589b28307c2c",
              "IPY_MODEL_0feee9a7781a47a99286fa524666ef14"
            ]
          }
        },
        "dcb253b6935e4ed0866c241cb44722a1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "36246122c68d4a20918f589b28307c2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6adab35daa6c478b9f17e0eebea57950",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 28,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 28,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bcefacf48e9a44e68eaec8ca0d77ac68"
          }
        },
        "0feee9a7781a47a99286fa524666ef14": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0a74baedea9240618e35ed0d637e5bc4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 28.0/28.0 [00:00&lt;00:00, 44.5B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4faea46ff4804f9eb0004f1b2133fe9c"
          }
        },
        "6adab35daa6c478b9f17e0eebea57950": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bcefacf48e9a44e68eaec8ca0d77ac68": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0a74baedea9240618e35ed0d637e5bc4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4faea46ff4804f9eb0004f1b2133fe9c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7be40d4c09f6401498ae4eac88fd8574": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_eca77e43f8634cc390dd58446dd54fd4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_338fe1549f4b41a880703f6c0420d670",
              "IPY_MODEL_bfe7d6aabd6d462e8db8acd92c63dd5e"
            ]
          }
        },
        "eca77e43f8634cc390dd58446dd54fd4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "338fe1549f4b41a880703f6c0420d670": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d26911da5d684bd597a762ddcace22aa",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 466062,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 466062,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3704f27c2084491799c43bd3fc203415"
          }
        },
        "bfe7d6aabd6d462e8db8acd92c63dd5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_dc6ec0d6798048af81ed6769d25228fa",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 466k/466k [00:00&lt;00:00, 2.36MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cd314367b9cd4e67b1bb05c4ed4b0d8a"
          }
        },
        "d26911da5d684bd597a762ddcace22aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3704f27c2084491799c43bd3fc203415": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dc6ec0d6798048af81ed6769d25228fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cd314367b9cd4e67b1bb05c4ed4b0d8a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ede58bafcad444a2bd70be6ac16f62f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_83a0f40709aa439d882f7946982e804d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e086a4136a874129af0fefe4fcae25ad",
              "IPY_MODEL_5b0b677b9541426fbe5dc0cc0e28d779"
            ]
          }
        },
        "83a0f40709aa439d882f7946982e804d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e086a4136a874129af0fefe4fcae25ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cb2097913c1d4640ab4b0bc124e472b5",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 570,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 570,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e1317a065ad549d3a64e4254a9ae17cb"
          }
        },
        "5b0b677b9541426fbe5dc0cc0e28d779": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6c87e70ff79549bb824f1066f8335687",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 570/570 [00:00&lt;00:00, 3.38kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1aada43c6f3e4e8592208fa593f3e37f"
          }
        },
        "cb2097913c1d4640ab4b0bc124e472b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e1317a065ad549d3a64e4254a9ae17cb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6c87e70ff79549bb824f1066f8335687": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1aada43c6f3e4e8592208fa593f3e37f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "073256f54ccd4c869c65bca87775eaac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7d541ae81f0b4276b91d03e80770fd89",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_60fbd3c32bb84f2b89d34040849cf343",
              "IPY_MODEL_474cd49b94114a3e99e81f2409420ee2"
            ]
          }
        },
        "7d541ae81f0b4276b91d03e80770fd89": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "60fbd3c32bb84f2b89d34040849cf343": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6511b1943ccb43e58f8c7420a443ba4b",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_52d46853959b4e72be3e6b4790d58f26"
          }
        },
        "474cd49b94114a3e99e81f2409420ee2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f0cfb556fbc14d57bf76ed161e0125b8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 440M/440M [00:09&lt;00:00, 47.7MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6057acb6296d433bb62230383843ae1b"
          }
        },
        "6511b1943ccb43e58f8c7420a443ba4b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "52d46853959b4e72be3e6b4790d58f26": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f0cfb556fbc14d57bf76ed161e0125b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6057acb6296d433bb62230383843ae1b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "91ab61c352ee46f9acaf523af32a199f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6e8811fffc5141dca299b692b30fa2ef",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5156801d249743998b5b511a529fde6a",
              "IPY_MODEL_87b78ea4734f4e47ad0747c3c8556b10"
            ]
          }
        },
        "6e8811fffc5141dca299b692b30fa2ef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5156801d249743998b5b511a529fde6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_bb5c08e60b594dda8caf2c83b25a3d0e",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 5,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 5,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7b0965a932bb4719a36f8987b118c856"
          }
        },
        "87b78ea4734f4e47ad0747c3c8556b10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d372df2cb94044fcaad555086000809d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 5/5 [25:24&lt;00:00, 304.97s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2c383efbadc34269bc995e62d7b20207"
          }
        },
        "bb5c08e60b594dda8caf2c83b25a3d0e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7b0965a932bb4719a36f8987b118c856": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d372df2cb94044fcaad555086000809d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2c383efbadc34269bc995e62d7b20207": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "405d40d550b4499e98facf55504bab2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_01c1f26d066d4a7d87e00ddff3ce9f9a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0aee0f8dc6c94aaebe03e7fd70c420bc",
              "IPY_MODEL_3e3ec8b4835f49778bfb260231a1965e"
            ]
          }
        },
        "01c1f26d066d4a7d87e00ddff3ce9f9a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0aee0f8dc6c94aaebe03e7fd70c420bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8ad02fd261a343dfb70865878c8bb217",
            "_dom_classes": [],
            "description": "Epoch 1: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 258,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 258,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_16c90145cf584773b25814c9af98ba3b"
          }
        },
        "3e3ec8b4835f49778bfb260231a1965e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2ecde1e6489540da87267eccad088e52",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 258/258 [05:05&lt;00:00,  1.09it/s, training_loss=0.416]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b94d47d7d43b4018a3cfb403348d5ad3"
          }
        },
        "8ad02fd261a343dfb70865878c8bb217": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "16c90145cf584773b25814c9af98ba3b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2ecde1e6489540da87267eccad088e52": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b94d47d7d43b4018a3cfb403348d5ad3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c471eb67045d44daa3492afc020f8566": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a34cc9cff655421cb53cd732866fce4a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a82c0eb0c1c34b229d206b2624326c12",
              "IPY_MODEL_f8d052657e2b4739b6ce963c738fd08d"
            ]
          }
        },
        "a34cc9cff655421cb53cd732866fce4a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a82c0eb0c1c34b229d206b2624326c12": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9ad8c5c051454258ba07c874e9eb99ec",
            "_dom_classes": [],
            "description": "Epoch 2: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 258,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 258,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_58b68a6f8bf94c22a995520fd9211c56"
          }
        },
        "f8d052657e2b4739b6ce963c738fd08d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_457e02bf106340908805a5826f4503a2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 258/258 [05:04&lt;00:00,  1.09it/s, training_loss=0.375]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3a500a0b7ce245c294fd9c07e797e1bf"
          }
        },
        "9ad8c5c051454258ba07c874e9eb99ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "58b68a6f8bf94c22a995520fd9211c56": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "457e02bf106340908805a5826f4503a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3a500a0b7ce245c294fd9c07e797e1bf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c04cb9f0f93a4d1f85ea04e308246e02": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_928037e2347741248e541bc33fa254c1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a558f25c41724f3281e26712a3341bb8",
              "IPY_MODEL_90c363dbd8a942bc91d5f96d17803a3d"
            ]
          }
        },
        "928037e2347741248e541bc33fa254c1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a558f25c41724f3281e26712a3341bb8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7199a108cc454afa9327f87a559b9c3f",
            "_dom_classes": [],
            "description": "Epoch 3: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 258,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 258,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_42ea682fdc1d48088c73ca1c21862723"
          }
        },
        "90c363dbd8a942bc91d5f96d17803a3d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d94bd25737f844b7923424171b27146c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 258/258 [05:04&lt;00:00,  1.08it/s, training_loss=0.423]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_574623dfa4a440a5893d2d9cdedb1319"
          }
        },
        "7199a108cc454afa9327f87a559b9c3f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "42ea682fdc1d48088c73ca1c21862723": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d94bd25737f844b7923424171b27146c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "574623dfa4a440a5893d2d9cdedb1319": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "66cd1e619ec84815815b8d0cd097767a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_31645944ca3042faa1114290322dff36",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_be1c43e9935342bc8b790d44f400461e",
              "IPY_MODEL_ea405905318c4d4ea9d1bb6fd50b15ca"
            ]
          }
        },
        "31645944ca3042faa1114290322dff36": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "be1c43e9935342bc8b790d44f400461e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f7aad5e5c6fb4c69b30c2100a0414787",
            "_dom_classes": [],
            "description": "Epoch 4: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 258,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 258,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3660fd69c600475d94d586f167815f07"
          }
        },
        "ea405905318c4d4ea9d1bb6fd50b15ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0c8f558591764849a82c8b5efe3020f5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 258/258 [05:05&lt;00:00,  1.09it/s, training_loss=0.338]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_004f3a55b8a5420b82c4f675ed088f0b"
          }
        },
        "f7aad5e5c6fb4c69b30c2100a0414787": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3660fd69c600475d94d586f167815f07": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0c8f558591764849a82c8b5efe3020f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "004f3a55b8a5420b82c4f675ed088f0b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "30005fe2b7524e9c8d55e902166a54b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_78be410dc50245159b28b89dbf3e774a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_15e9ba29b50b4ac9bbe62465c6ffade7",
              "IPY_MODEL_9528504c01614156a6bb25b4471fc7da"
            ]
          }
        },
        "78be410dc50245159b28b89dbf3e774a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "15e9ba29b50b4ac9bbe62465c6ffade7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d9953fad35c24b999ae69c4668aeb756",
            "_dom_classes": [],
            "description": "Epoch 5: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 258,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 258,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c2a34347615e4211836117522332354d"
          }
        },
        "9528504c01614156a6bb25b4471fc7da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d6999bcaf1504ab49fc06f521892be8a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 258/258 [05:04&lt;00:00,  1.08it/s, training_loss=0.355]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c36edf34c5f14beeacc63ee6cc76a351"
          }
        },
        "d9953fad35c24b999ae69c4668aeb756": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c2a34347615e4211836117522332354d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d6999bcaf1504ab49fc06f521892be8a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c36edf34c5f14beeacc63ee6cc76a351": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ashishpatel26/Covid19-tweet-sentimental-Analysis/blob/main/Transformer_for_Covid19_Tweet_classification.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A18wWmMmkGA1"
      },
      "source": [
        "### Install Transformer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tt3W77kJj-x7",
        "outputId": "7c76f1cf-709a-4ebe-8cee-b3e94e3c221c"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.8.2)\n",
            "Requirement already satisfied: huggingface-hub==0.0.12 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.12)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.45)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (4.6.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from transformers) (3.13)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from huggingface-hub==0.0.12->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.5.30)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DBtUD7Bjkayi"
      },
      "source": [
        "### **1.Load Framewoorks**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BP0oyhUDgcSH"
      },
      "source": [
        "import re\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "from tqdm.notebook import tqdm\n",
        "\n",
        "# Data Reading\n",
        "import torch\n",
        "from torch.utils.data import TensorDataset\n",
        "\n",
        "# NLP Task\n",
        "import transformers\n",
        "from transformers import BertTokenizer\n",
        "from transformers import BertForSequenceClassification\n",
        "\n",
        "# Define Seed\n",
        "import random\n",
        "\n",
        "seed_val = 42\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)\n",
        "device = torch.device('cuda')"
      ],
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rOxyOfo-kq33"
      },
      "source": [
        "### **2.Read dataset**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 439
        },
        "id": "rUuJoYp4jmtx",
        "outputId": "036d1d79-736d-4fad-ebcb-2947ee7f9983"
      },
      "source": [
        "train = pd.read_csv(\"https://raw.githubusercontent.com/ashishpatel26/Covid19-tweet-sentimental-Analysis/main/data/Corona_NLP_train.csv\", encoding='latin-1')\n",
        "test = pd.read_csv(\"https://raw.githubusercontent.com/ashishpatel26/Covid19-tweet-sentimental-Analysis/main/data/Corona_NLP_test.csv\", encoding='latin-1')\n",
        "print(\"--Train Data--\")\n",
        "display(train.head())\n",
        "print(\"--Test Data--\")\n",
        "display(test.head())"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--Train Data--\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>UserName</th>\n",
              "      <th>ScreenName</th>\n",
              "      <th>Location</th>\n",
              "      <th>TweetAt</th>\n",
              "      <th>OriginalTweet</th>\n",
              "      <th>Sentiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3799</td>\n",
              "      <td>48751</td>\n",
              "      <td>London</td>\n",
              "      <td>16-03-2020</td>\n",
              "      <td>@MeNyrbie @Phil_Gahan @Chrisitv https://t.co/i...</td>\n",
              "      <td>Neutral</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3800</td>\n",
              "      <td>48752</td>\n",
              "      <td>UK</td>\n",
              "      <td>16-03-2020</td>\n",
              "      <td>advice Talk to your neighbours family to excha...</td>\n",
              "      <td>Positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3801</td>\n",
              "      <td>48753</td>\n",
              "      <td>Vagabonds</td>\n",
              "      <td>16-03-2020</td>\n",
              "      <td>Coronavirus Australia: Woolworths to give elde...</td>\n",
              "      <td>Positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3802</td>\n",
              "      <td>48754</td>\n",
              "      <td>NaN</td>\n",
              "      <td>16-03-2020</td>\n",
              "      <td>My food stock is not the only one which is emp...</td>\n",
              "      <td>Positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3803</td>\n",
              "      <td>48755</td>\n",
              "      <td>NaN</td>\n",
              "      <td>16-03-2020</td>\n",
              "      <td>Me, ready to go at supermarket during the #COV...</td>\n",
              "      <td>Extremely Negative</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   UserName  ...           Sentiment\n",
              "0      3799  ...             Neutral\n",
              "1      3800  ...            Positive\n",
              "2      3801  ...            Positive\n",
              "3      3802  ...            Positive\n",
              "4      3803  ...  Extremely Negative\n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "--Test Data--\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>UserName</th>\n",
              "      <th>ScreenName</th>\n",
              "      <th>Location</th>\n",
              "      <th>TweetAt</th>\n",
              "      <th>OriginalTweet</th>\n",
              "      <th>Sentiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>44953</td>\n",
              "      <td>NYC</td>\n",
              "      <td>02-03-2020</td>\n",
              "      <td>TRENDING: New Yorkers encounter empty supermar...</td>\n",
              "      <td>Extremely Negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>44954</td>\n",
              "      <td>Seattle, WA</td>\n",
              "      <td>02-03-2020</td>\n",
              "      <td>When I couldn't find hand sanitizer at Fred Me...</td>\n",
              "      <td>Positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>44955</td>\n",
              "      <td>NaN</td>\n",
              "      <td>02-03-2020</td>\n",
              "      <td>Find out how you can protect yourself and love...</td>\n",
              "      <td>Extremely Positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>44956</td>\n",
              "      <td>Chicagoland</td>\n",
              "      <td>02-03-2020</td>\n",
              "      <td>#Panic buying hits #NewYork City as anxious sh...</td>\n",
              "      <td>Negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>44957</td>\n",
              "      <td>Melbourne, Victoria</td>\n",
              "      <td>03-03-2020</td>\n",
              "      <td>#toiletpaper #dunnypaper #coronavirus #coronav...</td>\n",
              "      <td>Neutral</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   UserName  ...           Sentiment\n",
              "0         1  ...  Extremely Negative\n",
              "1         2  ...            Positive\n",
              "2         3  ...  Extremely Positive\n",
              "3         4  ...            Negative\n",
              "4         5  ...             Neutral\n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t6SJRcK1lXJh"
      },
      "source": [
        "### **3.Data Preprocessing**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "44rsrAdJvFxR"
      },
      "source": [
        "##### **1.Mention and Hashtag**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ifFMGAf-k1kN"
      },
      "source": [
        "class MentionHastag():\n",
        "  '''\n",
        "  Extract the Mention and HashTag  \n",
        "  '''\n",
        "  def __init__(self, text = None):\n",
        "    self._text = text\n",
        "\n",
        "  def extractHashTag(self, text = None):\n",
        "    if text:\n",
        "      self.hashtag = re.findall(r'#(\\w+)', text)\n",
        "      return \" \".join(self.hashtag)\n",
        "    else:\n",
        "      self.hashtag = re.findall(r'#(\\w)', self._text)\n",
        "      return \" \".join(self.hashtag)\n",
        "\n",
        "  def extractMentions(self, text = None):\n",
        "    if text:\n",
        "      self.mention = re.findall(r'@(\\w+)', text)\n",
        "      return \" \".join(self.mention)\n",
        "    else:\n",
        "      self.mention = re.findall(r'@(\\w)', self._text)\n",
        "      return \" \".join(self.mention)"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tVKu6trenvZi"
      },
      "source": [
        "menhas = MentionHastag()"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ylHNmFJbn4zr"
      },
      "source": [
        "train['hashtags'] = train['OriginalTweet'].apply(lambda x : menhas.extractHashTag(text=x))\n",
        "train['mensions'] = train['OriginalTweet'].apply(lambda x : menhas.extractMentions(text=x))\n",
        "\n",
        "test['hashtags'] = test['OriginalTweet'].apply(lambda x : menhas.extractHashTag(text=x))\n",
        "test['mensions'] = test['OriginalTweet'].apply(lambda x : menhas.extractMentions(text=x))"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 210
        },
        "id": "7too7C5FoSsE",
        "outputId": "33841daa-d6f4-4f1e-823b-c378093d97a6"
      },
      "source": [
        "train.head()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>UserName</th>\n",
              "      <th>ScreenName</th>\n",
              "      <th>Location</th>\n",
              "      <th>TweetAt</th>\n",
              "      <th>OriginalTweet</th>\n",
              "      <th>Sentiment</th>\n",
              "      <th>hashtags</th>\n",
              "      <th>mensions</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3799</td>\n",
              "      <td>48751</td>\n",
              "      <td>London</td>\n",
              "      <td>16-03-2020</td>\n",
              "      <td>@MeNyrbie @Phil_Gahan @Chrisitv https://t.co/i...</td>\n",
              "      <td>Neutral</td>\n",
              "      <td></td>\n",
              "      <td>MeNyrbie Phil_Gahan Chrisitv</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3800</td>\n",
              "      <td>48752</td>\n",
              "      <td>UK</td>\n",
              "      <td>16-03-2020</td>\n",
              "      <td>advice Talk to your neighbours family to excha...</td>\n",
              "      <td>Positive</td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3801</td>\n",
              "      <td>48753</td>\n",
              "      <td>Vagabonds</td>\n",
              "      <td>16-03-2020</td>\n",
              "      <td>Coronavirus Australia: Woolworths to give elde...</td>\n",
              "      <td>Positive</td>\n",
              "      <td></td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3802</td>\n",
              "      <td>48754</td>\n",
              "      <td>NaN</td>\n",
              "      <td>16-03-2020</td>\n",
              "      <td>My food stock is not the only one which is emp...</td>\n",
              "      <td>Positive</td>\n",
              "      <td>COVID19france COVID_19 COVID19 coronavirus con...</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3803</td>\n",
              "      <td>48755</td>\n",
              "      <td>NaN</td>\n",
              "      <td>16-03-2020</td>\n",
              "      <td>Me, ready to go at supermarket during the #COV...</td>\n",
              "      <td>Extremely Negative</td>\n",
              "      <td>COVID19 coronavirus CoronavirusFrance restezch...</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   UserName  ...                      mensions\n",
              "0      3799  ...  MeNyrbie Phil_Gahan Chrisitv\n",
              "1      3800  ...                              \n",
              "2      3801  ...                              \n",
              "3      3802  ...                              \n",
              "4      3803  ...                              \n",
              "\n",
              "[5 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 210
        },
        "id": "U6bcm_jEoaOl",
        "outputId": "be56c058-f879-4913-b6d0-5b6c8aa4f53b"
      },
      "source": [
        "test.head()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>UserName</th>\n",
              "      <th>ScreenName</th>\n",
              "      <th>Location</th>\n",
              "      <th>TweetAt</th>\n",
              "      <th>OriginalTweet</th>\n",
              "      <th>Sentiment</th>\n",
              "      <th>hashtags</th>\n",
              "      <th>mensions</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>44953</td>\n",
              "      <td>NYC</td>\n",
              "      <td>02-03-2020</td>\n",
              "      <td>TRENDING: New Yorkers encounter empty supermar...</td>\n",
              "      <td>Extremely Negative</td>\n",
              "      <td>coronavirus</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>44954</td>\n",
              "      <td>Seattle, WA</td>\n",
              "      <td>02-03-2020</td>\n",
              "      <td>When I couldn't find hand sanitizer at Fred Me...</td>\n",
              "      <td>Positive</td>\n",
              "      <td>Amazon coronavirus</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>44955</td>\n",
              "      <td>NaN</td>\n",
              "      <td>02-03-2020</td>\n",
              "      <td>Find out how you can protect yourself and love...</td>\n",
              "      <td>Extremely Positive</td>\n",
              "      <td>coronavirus</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>44956</td>\n",
              "      <td>Chicagoland</td>\n",
              "      <td>02-03-2020</td>\n",
              "      <td>#Panic buying hits #NewYork City as anxious sh...</td>\n",
              "      <td>Negative</td>\n",
              "      <td>Panic NewYork healthcare BigApple coronavirus ...</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>44957</td>\n",
              "      <td>Melbourne, Victoria</td>\n",
              "      <td>03-03-2020</td>\n",
              "      <td>#toiletpaper #dunnypaper #coronavirus #coronav...</td>\n",
              "      <td>Neutral</td>\n",
              "      <td>toiletpaper dunnypaper coronavirus coronavirus...</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   UserName  ...  mensions\n",
              "0         1  ...          \n",
              "1         2  ...          \n",
              "2         3  ...          \n",
              "3         4  ...          \n",
              "4         5  ...          \n",
              "\n",
              "[5 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UJSiKfaNo65c"
      },
      "source": [
        "##### **2.Encoding Class**\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SRZxLYMuoy49"
      },
      "source": [
        "from sklearn.preprocessing import LabelEncoder\n",
        "le = LabelEncoder()"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ss68nDT9qd_0"
      },
      "source": [
        "train[\"encoded_sentiments\"] = le.fit_transform(train[\"Sentiment\"])\n",
        "test[\"encoded_sentiments\"] = le.fit_transform(test[\"Sentiment\"])"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OehC4Rdjs_ge"
      },
      "source": [
        "##### **3.Removing  Special Characters**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m99Qy1agq0i0"
      },
      "source": [
        "train[\"OriginalTweet\"] = train[\"OriginalTweet\"].apply(lambda x: ' '.join(re.sub(\"(@[A-Za-z0-9]+)| \\\n",
        "                                                                                ([^0-9A-Za-z \\t])| \\\n",
        "                                                                                (\\w+:\\/\\/\\S+)\",\" \",x) \\\n",
        "                                                                                .split())) \n",
        "test[\"OriginalTweet\"] = test[\"OriginalTweet\"].apply(lambda x: ' '.join(re.sub(\"(@[A-Za-z0-9]+)| \\\n",
        "                                                                                ([^0-9A-Za-z \\t])| \\\n",
        "                                                                                (\\w+:\\/\\/\\S+)\",\" \",x) \\\n",
        "                                                                                .split()))"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xQzFLhkluGSS"
      },
      "source": [
        "### **4. Train Test Split**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UNF3hh8kswRx",
        "outputId": "c1df512a-edf9-4f71-970b-c8d883c04daa"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "X_train, X_val, y_train, y_val = train_test_split(train[\"OriginalTweet\"], train[\"encoded_sentiments\"], test_size = 0.2)\n",
        "\n",
        "for name, var in zip(['X_train', 'X_val', 'y_train', 'y_val'], [X_train, X_val, y_train, y_val]):\n",
        "  print(f\"{name} shape : {var.shape}\")"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "X_train shape : (32925,)\n",
            "X_val shape : (8232,)\n",
            "y_train shape : (32925,)\n",
            "y_val shape : (8232,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lPnhsQgWu5rx"
      },
      "source": [
        "### **5. Bert Tokenizer**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 167,
          "referenced_widgets": [
            "d1fe038b94344df7a719b2471da9849b",
            "5dd68006717f4f7eaa589f80ac4f042f",
            "3c3e8383f81847d291a11ea06346784b",
            "2cbaa5a166ba4701a311b042e3a75dfe",
            "ef80ce6ca8064944aec4904394fcdb0a",
            "1a7c824f16744a80817727a94f915698",
            "bdeec54153854382ae33e9b42378c60d",
            "5a97e0a4bde24777a3c40392b5913b2e",
            "b7ab52517f744351a01bf50ac296ab0d",
            "dcb253b6935e4ed0866c241cb44722a1",
            "36246122c68d4a20918f589b28307c2c",
            "0feee9a7781a47a99286fa524666ef14",
            "6adab35daa6c478b9f17e0eebea57950",
            "bcefacf48e9a44e68eaec8ca0d77ac68",
            "0a74baedea9240618e35ed0d637e5bc4",
            "4faea46ff4804f9eb0004f1b2133fe9c",
            "7be40d4c09f6401498ae4eac88fd8574",
            "eca77e43f8634cc390dd58446dd54fd4",
            "338fe1549f4b41a880703f6c0420d670",
            "bfe7d6aabd6d462e8db8acd92c63dd5e",
            "d26911da5d684bd597a762ddcace22aa",
            "3704f27c2084491799c43bd3fc203415",
            "dc6ec0d6798048af81ed6769d25228fa",
            "cd314367b9cd4e67b1bb05c4ed4b0d8a"
          ]
        },
        "id": "whyr94yEszG4",
        "outputId": "94a6cb2a-fc29-49d6-a5e8-cdc630cfbab5"
      },
      "source": [
        "bert_tn = transformers.BertTokenizer.from_pretrained('bert-base-uncased',do_lower_case=True)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d1fe038b94344df7a719b2471da9849b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b7ab52517f744351a01bf50ac296ab0d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=28.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7be40d4c09f6401498ae4eac88fd8574",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=466062.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G4p4E9hnxrlE"
      },
      "source": [
        "encoding_train_data = bert_tn.batch_encode_plus(X_train,\n",
        "                                                add_special_tokens = True,\n",
        "                                                return_attention_mask = True,\n",
        "                                                padding=True,\n",
        "                                                max_length = 50, \n",
        "                                                truncation=True,\n",
        "                                                return_tensors='pt')\n",
        "encoding_val_data = bert_tn.batch_encode_plus(X_val,\n",
        "                                              add_special_tokens = True,\n",
        "                                              return_attention_mask = True,\n",
        "                                              padding=True,\n",
        "                                              max_length = 50,\n",
        "                                              truncation=True,\n",
        "                                              return_tensors='pt')"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tyacniBGz2no",
        "outputId": "698dafac-2824-48ba-d1b6-c17b96dc561e"
      },
      "source": [
        "from pprint import pprint\n",
        "pprint(encoding_train_data)"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'attention_mask': tensor([[1, 1, 1,  ..., 1, 1, 1],\n",
            "        [1, 1, 1,  ..., 0, 0, 0],\n",
            "        [1, 1, 1,  ..., 0, 0, 0],\n",
            "        ...,\n",
            "        [1, 1, 1,  ..., 1, 1, 1],\n",
            "        [1, 1, 1,  ..., 1, 1, 1],\n",
            "        [1, 1, 1,  ..., 0, 0, 0]]),\n",
            " 'input_ids': tensor([[  101,  2182,  3022,  ..., 16770,  1024,   102],\n",
            "        [  101,  2070,  2752,  ...,     0,     0,     0],\n",
            "        [  101,  1996, 12702,  ...,     0,     0,     0],\n",
            "        ...,\n",
            "        [  101,  2306,  1037,  ...,  7712,  1001,   102],\n",
            "        [  101,  1029,  1029,  ...,  2010,  5958,   102],\n",
            "        [  101, 13025,  3573,  ...,     0,     0,     0]]),\n",
            " 'token_type_ids': tensor([[0, 0, 0,  ..., 0, 0, 0],\n",
            "        [0, 0, 0,  ..., 0, 0, 0],\n",
            "        [0, 0, 0,  ..., 0, 0, 0],\n",
            "        ...,\n",
            "        [0, 0, 0,  ..., 0, 0, 0],\n",
            "        [0, 0, 0,  ..., 0, 0, 0],\n",
            "        [0, 0, 0,  ..., 0, 0, 0]])}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DlDGaSp61rsB"
      },
      "source": [
        "### **6.Visualize the tokenise data**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "wTIl-GpA0N09",
        "outputId": "51942dd5-a622-43e6-8bd7-38cffe29339d"
      },
      "source": [
        "for i in ['attention_mask','input_ids','token_type_ids']:\n",
        "  print(f\">>>---Tensor of {i}\")\n",
        "  display(pd.DataFrame(encoding_train_data[i].numpy()))"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            ">>>---Tensor of attention_mask\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "      <th>27</th>\n",
              "      <th>28</th>\n",
              "      <th>29</th>\n",
              "      <th>30</th>\n",
              "      <th>31</th>\n",
              "      <th>32</th>\n",
              "      <th>33</th>\n",
              "      <th>34</th>\n",
              "      <th>35</th>\n",
              "      <th>36</th>\n",
              "      <th>37</th>\n",
              "      <th>38</th>\n",
              "      <th>39</th>\n",
              "      <th>40</th>\n",
              "      <th>41</th>\n",
              "      <th>42</th>\n",
              "      <th>43</th>\n",
              "      <th>44</th>\n",
              "      <th>45</th>\n",
              "      <th>46</th>\n",
              "      <th>47</th>\n",
              "      <th>48</th>\n",
              "      <th>49</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32920</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32921</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32922</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32923</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32924</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>32925 rows × 50 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       0   1   2   3   4   5   6   7   8   ...  41  42  43  44  45  46  47  48  49\n",
              "0       1   1   1   1   1   1   1   1   1  ...   1   1   1   1   1   1   1   1   1\n",
              "1       1   1   1   1   1   1   1   1   1  ...   0   0   0   0   0   0   0   0   0\n",
              "2       1   1   1   1   1   1   1   1   1  ...   0   0   0   0   0   0   0   0   0\n",
              "3       1   1   1   1   1   1   1   1   1  ...   1   1   0   0   0   0   0   0   0\n",
              "4       1   1   1   1   1   1   1   1   1  ...   1   1   1   1   1   1   1   1   1\n",
              "...    ..  ..  ..  ..  ..  ..  ..  ..  ..  ...  ..  ..  ..  ..  ..  ..  ..  ..  ..\n",
              "32920   1   1   1   1   1   1   1   1   1  ...   1   1   1   1   1   1   1   1   1\n",
              "32921   1   1   1   1   1   1   1   1   1  ...   0   0   0   0   0   0   0   0   0\n",
              "32922   1   1   1   1   1   1   1   1   1  ...   1   1   1   1   1   1   1   1   1\n",
              "32923   1   1   1   1   1   1   1   1   1  ...   1   1   1   1   1   1   1   1   1\n",
              "32924   1   1   1   1   1   1   1   1   1  ...   0   0   0   0   0   0   0   0   0\n",
              "\n",
              "[32925 rows x 50 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            ">>>---Tensor of input_ids\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "      <th>27</th>\n",
              "      <th>28</th>\n",
              "      <th>29</th>\n",
              "      <th>30</th>\n",
              "      <th>31</th>\n",
              "      <th>32</th>\n",
              "      <th>33</th>\n",
              "      <th>34</th>\n",
              "      <th>35</th>\n",
              "      <th>36</th>\n",
              "      <th>37</th>\n",
              "      <th>38</th>\n",
              "      <th>39</th>\n",
              "      <th>40</th>\n",
              "      <th>41</th>\n",
              "      <th>42</th>\n",
              "      <th>43</th>\n",
              "      <th>44</th>\n",
              "      <th>45</th>\n",
              "      <th>46</th>\n",
              "      <th>47</th>\n",
              "      <th>48</th>\n",
              "      <th>49</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>101</td>\n",
              "      <td>2182</td>\n",
              "      <td>3022</td>\n",
              "      <td>1037</td>\n",
              "      <td>6302</td>\n",
              "      <td>2013</td>\n",
              "      <td>4174</td>\n",
              "      <td>2073</td>\n",
              "      <td>2111</td>\n",
              "      <td>6023</td>\n",
              "      <td>2024</td>\n",
              "      <td>2108</td>\n",
              "      <td>6380</td>\n",
              "      <td>21529</td>\n",
              "      <td>6067</td>\n",
              "      <td>2138</td>\n",
              "      <td>2122</td>\n",
              "      <td>2024</td>\n",
              "      <td>2025</td>\n",
              "      <td>29347</td>\n",
              "      <td>14416</td>\n",
              "      <td>20925</td>\n",
              "      <td>2050</td>\n",
              "      <td>5350</td>\n",
              "      <td>1012</td>\n",
              "      <td>1045</td>\n",
              "      <td>2228</td>\n",
              "      <td>2023</td>\n",
              "      <td>2003</td>\n",
              "      <td>2054</td>\n",
              "      <td>2561</td>\n",
              "      <td>25691</td>\n",
              "      <td>2964</td>\n",
              "      <td>3504</td>\n",
              "      <td>2066</td>\n",
              "      <td>1999</td>\n",
              "      <td>2049</td>\n",
              "      <td>2220</td>\n",
              "      <td>5711</td>\n",
              "      <td>1012</td>\n",
              "      <td>1001</td>\n",
              "      <td>21887</td>\n",
              "      <td>23350</td>\n",
              "      <td>1001</td>\n",
              "      <td>2522</td>\n",
              "      <td>17258</td>\n",
              "      <td>16147</td>\n",
              "      <td>16770</td>\n",
              "      <td>1024</td>\n",
              "      <td>102</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>101</td>\n",
              "      <td>2070</td>\n",
              "      <td>2752</td>\n",
              "      <td>1997</td>\n",
              "      <td>4361</td>\n",
              "      <td>2024</td>\n",
              "      <td>2200</td>\n",
              "      <td>18636</td>\n",
              "      <td>2005</td>\n",
              "      <td>3784</td>\n",
              "      <td>13025</td>\n",
              "      <td>6023</td>\n",
              "      <td>2066</td>\n",
              "      <td>7473</td>\n",
              "      <td>4671</td>\n",
              "      <td>1001</td>\n",
              "      <td>2522</td>\n",
              "      <td>17258</td>\n",
              "      <td>1035</td>\n",
              "      <td>2539</td>\n",
              "      <td>1001</td>\n",
              "      <td>3784</td>\n",
              "      <td>1001</td>\n",
              "      <td>2833</td>\n",
              "      <td>102</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>101</td>\n",
              "      <td>1996</td>\n",
              "      <td>12702</td>\n",
              "      <td>2015</td>\n",
              "      <td>1999</td>\n",
              "      <td>2026</td>\n",
              "      <td>2181</td>\n",
              "      <td>1997</td>\n",
              "      <td>2024</td>\n",
              "      <td>2725</td>\n",
              "      <td>3599</td>\n",
              "      <td>2008</td>\n",
              "      <td>1998</td>\n",
              "      <td>2024</td>\n",
              "      <td>2128</td>\n",
              "      <td>9331</td>\n",
              "      <td>2075</td>\n",
              "      <td>1996</td>\n",
              "      <td>19054</td>\n",
              "      <td>2302</td>\n",
              "      <td>3976</td>\n",
              "      <td>21857</td>\n",
              "      <td>2015</td>\n",
              "      <td>2030</td>\n",
              "      <td>2151</td>\n",
              "      <td>2060</td>\n",
              "      <td>26136</td>\n",
              "      <td>13043</td>\n",
              "      <td>3424</td>\n",
              "      <td>7325</td>\n",
              "      <td>5468</td>\n",
              "      <td>102</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>101</td>\n",
              "      <td>1045</td>\n",
              "      <td>2005</td>\n",
              "      <td>2028</td>\n",
              "      <td>2123</td>\n",
              "      <td>1005</td>\n",
              "      <td>1056</td>\n",
              "      <td>2228</td>\n",
              "      <td>6634</td>\n",
              "      <td>9343</td>\n",
              "      <td>2003</td>\n",
              "      <td>4072</td>\n",
              "      <td>2138</td>\n",
              "      <td>2028</td>\n",
              "      <td>2097</td>\n",
              "      <td>2203</td>\n",
              "      <td>2039</td>\n",
              "      <td>8840</td>\n",
              "      <td>18606</td>\n",
              "      <td>2035</td>\n",
              "      <td>2037</td>\n",
              "      <td>2833</td>\n",
              "      <td>2349</td>\n",
              "      <td>2000</td>\n",
              "      <td>7170</td>\n",
              "      <td>21146</td>\n",
              "      <td>4667</td>\n",
              "      <td>2061</td>\n",
              "      <td>2009</td>\n",
              "      <td>2003</td>\n",
              "      <td>2061</td>\n",
              "      <td>2025</td>\n",
              "      <td>4072</td>\n",
              "      <td>1012</td>\n",
              "      <td>1001</td>\n",
              "      <td>21887</td>\n",
              "      <td>23350</td>\n",
              "      <td>2003</td>\n",
              "      <td>22308</td>\n",
              "      <td>2007</td>\n",
              "      <td>2256</td>\n",
              "      <td>4641</td>\n",
              "      <td>102</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>101</td>\n",
              "      <td>2235</td>\n",
              "      <td>5661</td>\n",
              "      <td>1999</td>\n",
              "      <td>2710</td>\n",
              "      <td>2215</td>\n",
              "      <td>2019</td>\n",
              "      <td>1005</td>\n",
              "      <td>5057</td>\n",
              "      <td>3293</td>\n",
              "      <td>9278</td>\n",
              "      <td>4335</td>\n",
              "      <td>2565</td>\n",
              "      <td>1005</td>\n",
              "      <td>2096</td>\n",
              "      <td>1996</td>\n",
              "      <td>2924</td>\n",
              "      <td>1997</td>\n",
              "      <td>2710</td>\n",
              "      <td>1998</td>\n",
              "      <td>4642</td>\n",
              "      <td>16257</td>\n",
              "      <td>1999</td>\n",
              "      <td>10258</td>\n",
              "      <td>3686</td>\n",
              "      <td>2613</td>\n",
              "      <td>3776</td>\n",
              "      <td>7597</td>\n",
              "      <td>2000</td>\n",
              "      <td>4337</td>\n",
              "      <td>2522</td>\n",
              "      <td>17258</td>\n",
              "      <td>1011</td>\n",
              "      <td>2539</td>\n",
              "      <td>16770</td>\n",
              "      <td>1024</td>\n",
              "      <td>1013</td>\n",
              "      <td>1013</td>\n",
              "      <td>1056</td>\n",
              "      <td>1012</td>\n",
              "      <td>2522</td>\n",
              "      <td>1013</td>\n",
              "      <td>1054</td>\n",
              "      <td>2487</td>\n",
              "      <td>2361</td>\n",
              "      <td>2620</td>\n",
              "      <td>2595</td>\n",
              "      <td>2581</td>\n",
              "      <td>26291</td>\n",
              "      <td>102</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32920</th>\n",
              "      <td>101</td>\n",
              "      <td>2047</td>\n",
              "      <td>2265</td>\n",
              "      <td>1024</td>\n",
              "      <td>2012</td>\n",
              "      <td>14844</td>\n",
              "      <td>2298</td>\n",
              "      <td>2066</td>\n",
              "      <td>22017</td>\n",
              "      <td>5910</td>\n",
              "      <td>2050</td>\n",
              "      <td>7832</td>\n",
              "      <td>1024</td>\n",
              "      <td>8292</td>\n",
              "      <td>2571</td>\n",
              "      <td>2497</td>\n",
              "      <td>1001</td>\n",
              "      <td>21887</td>\n",
              "      <td>23350</td>\n",
              "      <td>7928</td>\n",
              "      <td>1010</td>\n",
              "      <td>17006</td>\n",
              "      <td>7357</td>\n",
              "      <td>1010</td>\n",
              "      <td>1001</td>\n",
              "      <td>8870</td>\n",
              "      <td>5302</td>\n",
              "      <td>13469</td>\n",
              "      <td>8273</td>\n",
              "      <td>4779</td>\n",
              "      <td>4920</td>\n",
              "      <td>2050</td>\n",
              "      <td>2713</td>\n",
              "      <td>1010</td>\n",
              "      <td>5365</td>\n",
              "      <td>2188</td>\n",
              "      <td>11058</td>\n",
              "      <td>1010</td>\n",
              "      <td>1001</td>\n",
              "      <td>2047</td>\n",
              "      <td>9028</td>\n",
              "      <td>9488</td>\n",
              "      <td>2869</td>\n",
              "      <td>19237</td>\n",
              "      <td>1029</td>\n",
              "      <td>4942</td>\n",
              "      <td>1013</td>\n",
              "      <td>21469</td>\n",
              "      <td>1006</td>\n",
              "      <td>102</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32921</th>\n",
              "      <td>101</td>\n",
              "      <td>1999</td>\n",
              "      <td>8780</td>\n",
              "      <td>2015</td>\n",
              "      <td>8704</td>\n",
              "      <td>2000</td>\n",
              "      <td>3143</td>\n",
              "      <td>2048</td>\n",
              "      <td>2192</td>\n",
              "      <td>2624</td>\n",
              "      <td>25090</td>\n",
              "      <td>6290</td>\n",
              "      <td>4264</td>\n",
              "      <td>1999</td>\n",
              "      <td>1996</td>\n",
              "      <td>2866</td>\n",
              "      <td>1998</td>\n",
              "      <td>2762</td>\n",
              "      <td>1999</td>\n",
              "      <td>2184</td>\n",
              "      <td>2420</td>\n",
              "      <td>2000</td>\n",
              "      <td>3965</td>\n",
              "      <td>1015</td>\n",
              "      <td>2213</td>\n",
              "      <td>11015</td>\n",
              "      <td>2566</td>\n",
              "      <td>3204</td>\n",
              "      <td>2169</td>\n",
              "      <td>2000</td>\n",
              "      <td>4769</td>\n",
              "      <td>1037</td>\n",
              "      <td>4187</td>\n",
              "      <td>15843</td>\n",
              "      <td>2408</td>\n",
              "      <td>2885</td>\n",
              "      <td>102</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32922</th>\n",
              "      <td>101</td>\n",
              "      <td>2306</td>\n",
              "      <td>1037</td>\n",
              "      <td>2733</td>\n",
              "      <td>1010</td>\n",
              "      <td>9871</td>\n",
              "      <td>11670</td>\n",
              "      <td>1010</td>\n",
              "      <td>4744</td>\n",
              "      <td>6853</td>\n",
              "      <td>1998</td>\n",
              "      <td>13025</td>\n",
              "      <td>3573</td>\n",
              "      <td>5126</td>\n",
              "      <td>2150</td>\n",
              "      <td>2062</td>\n",
              "      <td>2590</td>\n",
              "      <td>2008</td>\n",
              "      <td>3297</td>\n",
              "      <td>2111</td>\n",
              "      <td>1001</td>\n",
              "      <td>12609</td>\n",
              "      <td>1001</td>\n",
              "      <td>18402</td>\n",
              "      <td>1001</td>\n",
              "      <td>21887</td>\n",
              "      <td>23350</td>\n",
              "      <td>1001</td>\n",
              "      <td>2522</td>\n",
              "      <td>17258</td>\n",
              "      <td>16147</td>\n",
              "      <td>1001</td>\n",
              "      <td>21887</td>\n",
              "      <td>23350</td>\n",
              "      <td>5833</td>\n",
              "      <td>23890</td>\n",
              "      <td>1001</td>\n",
              "      <td>21887</td>\n",
              "      <td>6873</td>\n",
              "      <td>25778</td>\n",
              "      <td>22571</td>\n",
              "      <td>3366</td>\n",
              "      <td>1001</td>\n",
              "      <td>21887</td>\n",
              "      <td>23350</td>\n",
              "      <td>9739</td>\n",
              "      <td>3207</td>\n",
              "      <td>7712</td>\n",
              "      <td>1001</td>\n",
              "      <td>102</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32923</th>\n",
              "      <td>101</td>\n",
              "      <td>1029</td>\n",
              "      <td>1029</td>\n",
              "      <td>2040</td>\n",
              "      <td>11923</td>\n",
              "      <td>1012</td>\n",
              "      <td>3504</td>\n",
              "      <td>2066</td>\n",
              "      <td>2057</td>\n",
              "      <td>2179</td>\n",
              "      <td>1008</td>\n",
              "      <td>2178</td>\n",
              "      <td>1008</td>\n",
              "      <td>5205</td>\n",
              "      <td>2040</td>\n",
              "      <td>1000</td>\n",
              "      <td>3402</td>\n",
              "      <td>1000</td>\n",
              "      <td>14019</td>\n",
              "      <td>2010</td>\n",
              "      <td>4518</td>\n",
              "      <td>9583</td>\n",
              "      <td>2157</td>\n",
              "      <td>2044</td>\n",
              "      <td>1037</td>\n",
              "      <td>2797</td>\n",
              "      <td>27918</td>\n",
              "      <td>2006</td>\n",
              "      <td>1001</td>\n",
              "      <td>21887</td>\n",
              "      <td>23350</td>\n",
              "      <td>1012</td>\n",
              "      <td>5205</td>\n",
              "      <td>2853</td>\n",
              "      <td>2039</td>\n",
              "      <td>2000</td>\n",
              "      <td>1002</td>\n",
              "      <td>3156</td>\n",
              "      <td>1010</td>\n",
              "      <td>2199</td>\n",
              "      <td>1997</td>\n",
              "      <td>2010</td>\n",
              "      <td>4518</td>\n",
              "      <td>2006</td>\n",
              "      <td>1996</td>\n",
              "      <td>6928</td>\n",
              "      <td>2044</td>\n",
              "      <td>2010</td>\n",
              "      <td>5958</td>\n",
              "      <td>102</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32924</th>\n",
              "      <td>101</td>\n",
              "      <td>13025</td>\n",
              "      <td>3573</td>\n",
              "      <td>2448</td>\n",
              "      <td>3201</td>\n",
              "      <td>1029</td>\n",
              "      <td>1020</td>\n",
              "      <td>6199</td>\n",
              "      <td>10814</td>\n",
              "      <td>11022</td>\n",
              "      <td>1001</td>\n",
              "      <td>2591</td>\n",
              "      <td>10521</td>\n",
              "      <td>5794</td>\n",
              "      <td>6129</td>\n",
              "      <td>16770</td>\n",
              "      <td>1024</td>\n",
              "      <td>1013</td>\n",
              "      <td>1013</td>\n",
              "      <td>1056</td>\n",
              "      <td>1012</td>\n",
              "      <td>2522</td>\n",
              "      <td>1013</td>\n",
              "      <td>1057</td>\n",
              "      <td>26224</td>\n",
              "      <td>2290</td>\n",
              "      <td>4160</td>\n",
              "      <td>15088</td>\n",
              "      <td>2015</td>\n",
              "      <td>2615</td>\n",
              "      <td>2361</td>\n",
              "      <td>102</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>32925 rows × 50 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "        0      1      2     3      4   ...     45     46     47     48   49\n",
              "0      101   2182   3022  1037   6302  ...  17258  16147  16770   1024  102\n",
              "1      101   2070   2752  1997   4361  ...      0      0      0      0    0\n",
              "2      101   1996  12702  2015   1999  ...      0      0      0      0    0\n",
              "3      101   1045   2005  2028   2123  ...      0      0      0      0    0\n",
              "4      101   2235   5661  1999   2710  ...   2620   2595   2581  26291  102\n",
              "...    ...    ...    ...   ...    ...  ...    ...    ...    ...    ...  ...\n",
              "32920  101   2047   2265  1024   2012  ...   4942   1013  21469   1006  102\n",
              "32921  101   1999   8780  2015   8704  ...      0      0      0      0    0\n",
              "32922  101   2306   1037  2733   1010  ...   9739   3207   7712   1001  102\n",
              "32923  101   1029   1029  2040  11923  ...   6928   2044   2010   5958  102\n",
              "32924  101  13025   3573  2448   3201  ...      0      0      0      0    0\n",
              "\n",
              "[32925 rows x 50 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            ">>>---Tensor of token_type_ids\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "      <th>27</th>\n",
              "      <th>28</th>\n",
              "      <th>29</th>\n",
              "      <th>30</th>\n",
              "      <th>31</th>\n",
              "      <th>32</th>\n",
              "      <th>33</th>\n",
              "      <th>34</th>\n",
              "      <th>35</th>\n",
              "      <th>36</th>\n",
              "      <th>37</th>\n",
              "      <th>38</th>\n",
              "      <th>39</th>\n",
              "      <th>40</th>\n",
              "      <th>41</th>\n",
              "      <th>42</th>\n",
              "      <th>43</th>\n",
              "      <th>44</th>\n",
              "      <th>45</th>\n",
              "      <th>46</th>\n",
              "      <th>47</th>\n",
              "      <th>48</th>\n",
              "      <th>49</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32920</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32921</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32922</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32923</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32924</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>32925 rows × 50 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       0   1   2   3   4   5   6   7   8   ...  41  42  43  44  45  46  47  48  49\n",
              "0       0   0   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   0   0   0\n",
              "1       0   0   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   0   0   0\n",
              "2       0   0   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   0   0   0\n",
              "3       0   0   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   0   0   0\n",
              "4       0   0   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   0   0   0\n",
              "...    ..  ..  ..  ..  ..  ..  ..  ..  ..  ...  ..  ..  ..  ..  ..  ..  ..  ..  ..\n",
              "32920   0   0   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   0   0   0\n",
              "32921   0   0   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   0   0   0\n",
              "32922   0   0   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   0   0   0\n",
              "32923   0   0   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   0   0   0\n",
              "32924   0   0   0   0   0   0   0   0   0  ...   0   0   0   0   0   0   0   0   0\n",
              "\n",
              "[32925 rows x 50 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JmfjjyFL179W"
      },
      "source": [
        "### **7.Extract the Encoded data**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yj7sa8qj09Jq"
      },
      "source": [
        "train_input_ids = encoding_train_data['input_ids']\n",
        "train_attention_masks = encoding_train_data['attention_mask']\n",
        "train_labels = torch.tensor(y_train.values)\n",
        "\n",
        "\n",
        "val_input_ids = encoding_val_data['input_ids']\n",
        "val_attention_masks = encoding_val_data['attention_mask']\n",
        "val_labels = torch.tensor(y_val.values)"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k9y0IEyg2r9Z"
      },
      "source": [
        "### **8.Convert Pytorch Tensor Dataset Format**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x5te-u-K2gvc"
      },
      "source": [
        "# Pytorch TensorDataset Instance\n",
        "ds_train = TensorDataset(train_input_ids, train_attention_masks, train_labels)\n",
        "ds_Val = TensorDataset(val_input_ids, val_attention_masks, val_labels)"
      ],
      "execution_count": 56,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6JCchgME3LME"
      },
      "source": [
        "### **9.Bert Model Initialization**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 227,
          "referenced_widgets": [
            "ede58bafcad444a2bd70be6ac16f62f6",
            "83a0f40709aa439d882f7946982e804d",
            "e086a4136a874129af0fefe4fcae25ad",
            "5b0b677b9541426fbe5dc0cc0e28d779",
            "cb2097913c1d4640ab4b0bc124e472b5",
            "e1317a065ad549d3a64e4254a9ae17cb",
            "6c87e70ff79549bb824f1066f8335687",
            "1aada43c6f3e4e8592208fa593f3e37f",
            "073256f54ccd4c869c65bca87775eaac",
            "7d541ae81f0b4276b91d03e80770fd89",
            "60fbd3c32bb84f2b89d34040849cf343",
            "474cd49b94114a3e99e81f2409420ee2",
            "6511b1943ccb43e58f8c7420a443ba4b",
            "52d46853959b4e72be3e6b4790d58f26",
            "f0cfb556fbc14d57bf76ed161e0125b8",
            "6057acb6296d433bb62230383843ae1b"
          ]
        },
        "id": "ndFrNwQ329mT",
        "outputId": "64573766-de1b-4438-d918-e5dca8580227"
      },
      "source": [
        "bert = transformers.BertForSequenceClassification.from_pretrained('bert-base-uncased', \n",
        "                                                                  num_labels=5,\n",
        "                                                                  output_attentions=False,\n",
        "                                                                  output_hidden_states=False)"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ede58bafcad444a2bd70be6ac16f62f6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=570.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "073256f54ccd4c869c65bca87775eaac",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=440473133.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EDY6-J6o4V8i"
      },
      "source": [
        "### **10.DataLoader**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "exPerWxY3Fmu"
      },
      "source": [
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
        "\n",
        "train_dl =  DataLoader(ds_train, sampler = RandomSampler(ds_train), batch_size=128)\n",
        "val_dl =  DataLoader(ds_Val, sampler = RandomSampler(ds_Val), batch_size=128)"
      ],
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uWoL_RQA44Pp"
      },
      "source": [
        "### **11.Model Optimization**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hrKQRXbI40Hq"
      },
      "source": [
        "from transformers import AdamW, get_linear_schedule_with_warmup\n",
        "\n",
        "optimizer = AdamW(bert.parameters(), lr= 1e-5, eps = 1e-8)\n",
        "epochs = 5\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer=optimizer, num_warmup_steps=0, num_training_steps=len(train_dl))"
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tFcPybqi5xU3"
      },
      "source": [
        "### **12.Metrics**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TtDW8Roi5gVo"
      },
      "source": [
        "from sklearn.metrics import f1_score\n",
        "\n",
        "def f1_score_fn(actual, predicted):\n",
        "  preds = np.argmax(predicted, axis=1).flatten()\n",
        "  labels = actual.flatten()\n",
        "  return f1_score(labels, preds, average='weighted')"
      ],
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R5_rNFOz6gW1"
      },
      "source": [
        "### 13.Model Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Uf_EdAPc6UYp",
        "outputId": "00c1240b-1bd3-438e-f025-f2dd79c899ef"
      },
      "source": [
        "bert.to(device)"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertForSequenceClassification(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=5, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 157,
          "referenced_widgets": [
            "91ab61c352ee46f9acaf523af32a199f",
            "6e8811fffc5141dca299b692b30fa2ef",
            "5156801d249743998b5b511a529fde6a",
            "87b78ea4734f4e47ad0747c3c8556b10",
            "bb5c08e60b594dda8caf2c83b25a3d0e",
            "7b0965a932bb4719a36f8987b118c856",
            "d372df2cb94044fcaad555086000809d",
            "2c383efbadc34269bc995e62d7b20207",
            "405d40d550b4499e98facf55504bab2b",
            "01c1f26d066d4a7d87e00ddff3ce9f9a",
            "0aee0f8dc6c94aaebe03e7fd70c420bc",
            "3e3ec8b4835f49778bfb260231a1965e",
            "8ad02fd261a343dfb70865878c8bb217",
            "16c90145cf584773b25814c9af98ba3b",
            "2ecde1e6489540da87267eccad088e52",
            "b94d47d7d43b4018a3cfb403348d5ad3",
            "c471eb67045d44daa3492afc020f8566",
            "a34cc9cff655421cb53cd732866fce4a",
            "a82c0eb0c1c34b229d206b2624326c12",
            "f8d052657e2b4739b6ce963c738fd08d",
            "9ad8c5c051454258ba07c874e9eb99ec",
            "58b68a6f8bf94c22a995520fd9211c56",
            "457e02bf106340908805a5826f4503a2",
            "3a500a0b7ce245c294fd9c07e797e1bf",
            "c04cb9f0f93a4d1f85ea04e308246e02",
            "928037e2347741248e541bc33fa254c1",
            "a558f25c41724f3281e26712a3341bb8",
            "90c363dbd8a942bc91d5f96d17803a3d",
            "7199a108cc454afa9327f87a559b9c3f",
            "42ea682fdc1d48088c73ca1c21862723",
            "d94bd25737f844b7923424171b27146c",
            "574623dfa4a440a5893d2d9cdedb1319",
            "66cd1e619ec84815815b8d0cd097767a",
            "31645944ca3042faa1114290322dff36",
            "be1c43e9935342bc8b790d44f400461e",
            "ea405905318c4d4ea9d1bb6fd50b15ca",
            "f7aad5e5c6fb4c69b30c2100a0414787",
            "3660fd69c600475d94d586f167815f07",
            "0c8f558591764849a82c8b5efe3020f5",
            "004f3a55b8a5420b82c4f675ed088f0b",
            "30005fe2b7524e9c8d55e902166a54b4",
            "78be410dc50245159b28b89dbf3e774a",
            "15e9ba29b50b4ac9bbe62465c6ffade7",
            "9528504c01614156a6bb25b4471fc7da",
            "d9953fad35c24b999ae69c4668aeb756",
            "c2a34347615e4211836117522332354d",
            "d6999bcaf1504ab49fc06f521892be8a",
            "c36edf34c5f14beeacc63ee6cc76a351"
          ]
        },
        "id": "g_dc5rDw6pf1",
        "outputId": "f4124f24-741f-45bf-bd81-e23a0bc3a229"
      },
      "source": [
        "for epoch in tqdm(range(1, epochs+1)):\n",
        "  \n",
        "  # Model training\n",
        "  bert.train()\n",
        "\n",
        "  # loss count\n",
        "  train_loss_total = 0\n",
        "\n",
        "  # progress bar\n",
        "  progress_bar = tqdm(train_dl, desc=f'Epoch {epoch:1d}', leave = False, disable = False)\n",
        "\n",
        "  # Epochs\n",
        "  tqdm.write(f'\\nEpoch {epoch}')\n",
        "\n",
        "  for batch in progress_bar:\n",
        "\n",
        "    # make model gradient reset\n",
        "    bert.zero_grad()\n",
        "\n",
        "    # inputs\n",
        "    inputs = {\n",
        "              'input_ids' : batch[0].to(device),\n",
        "              'attention_mask' : batch[1].to(device),\n",
        "              'labels' : batch[2].to(device)\n",
        "              } \n",
        "\n",
        "    # outputs\n",
        "    outputs = bert(**inputs)\n",
        "\n",
        "    loss = outputs[0]\n",
        "    train_loss_total += loss.item()\n",
        "\n",
        "    # Backward propagation\n",
        "    loss.backward()\n",
        "\n",
        "    # clip the gradient\n",
        "    torch.nn.utils.clip_grad_norm_(bert.parameters(), 1.0)\n",
        "\n",
        "    # optimization bert\n",
        "    optimizer.step()\n",
        "\n",
        "    # scheduler\n",
        "    scheduler.step()\n",
        "\n",
        "    progress_bar.set_postfix({'training_loss': '{:.3f}'.format(loss.item()/len(batch))})\n",
        "\n",
        "    \n",
        "\n",
        "    train_loss_avg = train_loss_total/len(train_dl)\n",
        "\n",
        "  tqdm.write(f'Training loss: {train_loss_avg}')"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "91ab61c352ee46f9acaf523af32a199f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=5.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "405d40d550b4499e98facf55504bab2b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Epoch 1', max=258.0, style=ProgressStyle(description_widt…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r\r\r\r\r\r\r\r\r\r\r\r\rTraining loss: 1.3144335746765137\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c471eb67045d44daa3492afc020f8566",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Epoch 2', max=258.0, style=ProgressStyle(description_widt…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r\r\r\r\r\r\r\r\r\r\r\r\rTraining loss: 1.1967255902844807\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c04cb9f0f93a4d1f85ea04e308246e02",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Epoch 3', max=258.0, style=ProgressStyle(description_widt…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r\r\r\r\r\r\r\r\r\r\r\r\rTraining loss: 1.197342760341112\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "66cd1e619ec84815815b8d0cd097767a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Epoch 4', max=258.0, style=ProgressStyle(description_widt…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r\r\r\r\r\r\r\r\r\r\r\r\rTraining loss: 1.1972413206285284\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "30005fe2b7524e9c8d55e902166a54b4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Epoch 5', max=258.0, style=ProgressStyle(description_widt…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r\r\r\r\r\r\r\r\r\r\r\r\rTraining loss: 1.1965538646823677\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "swCSeGG_9dCQ"
      },
      "source": [
        "### **14.Model Evaluation**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xTFkwRD-8e6I"
      },
      "source": [
        "def evaluate(val_dl):\n",
        "\n",
        "  bert.eval()\n",
        "  \n",
        "  total_val_loss = 0\n",
        "  predictions, true_vals = [], []\n",
        "  \n",
        "  for batch in val_dl:\n",
        "      \n",
        "      batch = tuple(b.to(device) for b in batch)\n",
        "      \n",
        "      inputs ={\n",
        "                'input_ids':      batch[0],\n",
        "                'attention_mask': batch[1],\n",
        "                'labels':         batch[2]\n",
        "              }\n",
        "\n",
        "      with torch.no_grad():        \n",
        "          outputs = bert(**inputs)\n",
        "          \n",
        "      loss = outputs[0]\n",
        "      logits = outputs[1]\n",
        "      total_val_loss += loss.item()\n",
        "\n",
        "      logits = logits.detach().cpu().numpy()\n",
        "      label_ids = inputs['labels'].cpu().numpy()\n",
        "      predictions.append(logits)\n",
        "      true_vals.append(label_ids)\n",
        "  \n",
        "  val_avg_loss = total_val_loss/len(val_dl) \n",
        "  \n",
        "  predictions = np.concatenate(predictions, axis=0)\n",
        "  true_vals = np.concatenate(true_vals, axis=0)\n",
        "          \n",
        "  return val_avg_loss, predictions, true_vals"
      ],
      "execution_count": 58,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OiLbiNfE-C7Z"
      },
      "source": [
        "val_loss, predictions, true_vals = evaluate(val_dl = val_dl)"
      ],
      "execution_count": 60,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RwNqHdWyETYI",
        "outputId": "7c0256af-e67b-4606-8b06-4e4296a7cf4b"
      },
      "source": [
        "val_f1 = f1_score_fn(true_vals, predictions)\n",
        "print('Validation Loss = ', val_loss)\n",
        "print('Validiation F1 = ', val_f1)"
      ],
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Validation Loss =  1.1935951618047862\n",
            "Validiation F1 =  0.5040421392615076\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tli6GPqSFzjZ"
      },
      "source": [
        "### **15.Evalution Result**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c1fNMZRDDD70"
      },
      "source": [
        "encoded_classes = le.classes_\n",
        "predicted_category = [encoded_classes[np.argmax(x)] for x in predictions]\n",
        "true_category = [encoded_classes[x] for x in true_vals]"
      ],
      "execution_count": 65,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DezIl1h7Et29",
        "outputId": "34c84cde-8da7-46a5-88b4-b3c76f076cc9"
      },
      "source": [
        "x = 0\n",
        "for i in range(len(true_category)):\n",
        "    if true_category[i] == predicted_category[i]:\n",
        "        x += 1\n",
        "        \n",
        "print('Accuracy Score = ', x / len(true_category))"
      ],
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy Score =  0.5042517006802721\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fgl4tyltFJKs"
      },
      "source": [
        "from sklearn.metrics import confusion_matrix\n",
        "confusion_mat = confusion_matrix(y_true = true_category, y_pred = predicted_category, labels=list(encoded_classes))"
      ],
      "execution_count": 67,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 700
        },
        "id": "bGFgcFrrFN01",
        "outputId": "f0b03678-189a-445f-e53a-5b18e1663a3b"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "plt.figure(figsize=(15,12))\n",
        "df = pd.DataFrame(confusion_mat, index = list(encoded_classes),columns = list(encoded_classes))\n",
        "sns.heatmap(df, annot=True, fmt='.1f')\n",
        "plt.show()"
      ],
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAycAAAKrCAYAAADrrPK+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd5hU5fXA8e/ZXcrSi1SxixiNSBQVFUUFReyJJXY0Jphg7NGoWJLYjSVRE2P7xZpYookNC6Ii9q7YARWlgwJLL7vv749dEVxmGXFndmS/n+eZZ+feuffOuZjszpnznveNlBKSJEmSVNeK6joASZIkSQKTE0mSJEkFwuREkiRJUkEwOZEkSZJUEExOJEmSJBWEkly/wayj+zkdmKppe+cHdR2CCtClHXeu6xBUYH4/+em6DkEFqGnDxnUdggrUrDljo65jyMbi6Z/U6efjBmusX7D/TlZOJEmSJBUEkxNJkiRJBcHkRJIkSVJByHnPiSRJkqRlVJTXdQQFy8qJJEmSpIJg5USSJEnKp1RR1xEULCsnkiRJkgqCyYkkSZKkguCwLkmSJCmfKhzWlYmVE0mSJEkFweREkiRJyqOUKur0sTIR8X8RMTUi3l1mX5uIGBYRo6t+tq7aHxFxdUSMiYh3ImKLZc4ZWHX86IgYmM2/jcmJJEmSpGXdAuz+rX1nAMNTSl2B4VXbAAOArlWPQcB1UJnMAOcB2wBbA+d9ndDUxOREkiRJ0lIppWeBr761e1/g1qrntwL7LbP/tlTpJaBVRHQC+gPDUkpfpZRmAMOonvBUY0O8JEmSlE8/zIb4DimlSVXPJwMdqp6vCXyxzHHjq/Zl2l8jKyeSJElSPRIRgyLitWUeg77L+SmlBKRcxGblRJIkScqnOl4hPqV0A3DDdzxtSkR0SilNqhq2NbVq/wRgrWWO61K1bwKw07f2P7OyN7FyIkmSJGllHgS+nnFrIPDAMvuPrJq1qxcwq2r41+PAbhHRuqoRfreqfTWyciJJkiRpqYj4N5VVjzUiYjyVs25dAtwTEccA44CDqg4fCuwBjAHmAUcDpJS+iojzgVerjvtTSunbTfbVmJxIkiRJ+VRRXtcR1CildEiGl/qu4NgEHJfhOv8H/N93eW+HdUmSJEkqCFZOJEmSpHyq44b4QmblRJIkSVJBMDmRJEmSVBAc1iVJkiTl0w9zhfi8sHIiSZIkqSBYOZEkSZLyKNkQn5GVE0mSJEkFweREkiRJUkFwWJckSZKUTzbEZ2TlRJIkSVJBMDmRJEmSVBAc1iVJkiTlk7N1ZWTlRJIkSVJBsHIiSZIk5VNFeV1HULCsnEiSJEkqCCYnkiRJkgqCw7okSZKkfLIhPiMrJ5IkSZIKgpUTSZIkKZ9cIT4jKyeSJEmSCoLJiSRJkqSC4LAuSZIkKZ9siM/IyokkSZKkgmDlRJIkSconG+IzsnIiSZIkqSCYnEiSJEkqCA7rkiRJkvIopfK6DqFgWTmRJEmSVBCsnEiSJEn55FTCGVk5kSRJklQQTE4kSZIkFQSHdUmSJEn55DonGVk5kSRJklQQrJxIkiRJ+WRDfEZWTiRJkiQVBJMTSZIkSQXBYV2SJElSPlW4QnwmVk4kSZIkFQSTE0mSJEkFwWFdkiRJUj45W1dGJiffR2lTmhx9KkVd1oWUmP9/l5MWLaJ04ElEgwak8nIW3H415Z9+RHG3zWl6wp+omD4JgMWvP8fCB++odslYoyNNfj2EaNaC8nGjmX/DJVC+JM83plw4/rfHcMwxhxIR3Hzzv7j6mpuqHXPVlX9iwO67MG/+fI455mTefOvdOohUtam4UQN+fu/ZFDcsoaikmNFDX+GFK+8HYPvTDmSjPbcmlVfw9h3DefOfTwDQpdeP2Pm8wylqUMz8r2Zzz0EXVrtui7Xasde1x9G4dXOmjvqUoSddR8VixzCvLvx9oa8VFRUxYuT/mDhxCj8/8Fc8+sRdNGvWFIB27dry+mvvcNghv6523iGH/ozTTj8OgD9f9jf+/a/78xq3tKpMTr6H0sOOY/G7r7L473+C4hJo2Igmg89h4QO3sWTUq5R035rGBw1i7qWnArDk41HM++vZNV6z8YG/YtET97H4lWdofOSJNNxxAIuefigft6Mc2nTTbhxzzKFsu92eLFq0mKEP38kjQ59k7NjPlh4zYPdd6Lrhemy8SW+22XoL/nbtxWzXe++6C1q1onzhYu49+CIWz1tIUUkxB993Dp8+/TZtNlyT5p3b8M+dT4eUKG3bAoBGLZrQ78KjuO+Iy5g98cul+79txzMP5vWbHuOjh16i30VHs9nPd+LtO4bn89aUI/6+0LJ+M/goPvpoLM2bNwNgwG4HL33t9jv/xiMPP1ntnNatW3LGmcez0477kVJixMgHeHTok8ycWZa3uLUSrhCfkT0nq6q0KSUbbcbiZx+t3C5fAvPnAhClTZf+rJj55Xe6bMmPerD4tWcBWPz8E5RssX3txaw6s/HGXXnllTeZP38B5eXlPDvyJX6634Dljtl77/7cfud/AHj5lTdo2aolHTu2r4twVcsWz1sIQFFJMUUlJaQEmx/Rl5f+8j9ICYD5X1Z+aNh43+0Y/eirzJ745XL7v23t7Tbh46GvAPDef0ayYf8tc30byhN/X+hrnTt3pP/uO3PbrfdUe61582bsuOO2PPLwsGqv7dJvR55++nlmzJjFzJllPP308/TdtU8+Qpa+t5UmJxGxUUQMj4h3q7a7R0TNX//XA0VrdKRi9ixKjzmNZn/4B6VHnwING7PgX3+n8UGDaH7Fv2j882NZ8J9vSvHFG25Csz9eT5OTL6Ko8zrVrhnNWpDmzVmaTVfMmE5Rq7Z5uyflznvvfUjv3tvQpk1rSksbM2D3XejSpfNyx6zZuSPjv5i4dHvC+Ems2bljvkNVDkRRcMSjF/KbN//OuOdGMfmtsbRapz3d9t6Gwx7+Ez+79TRardsBgNbrd6Rxy6YcdPcQDn/kfDbZv3e165W2bsaCsnmk8srfFXMmfUWzjq3zek/KHX9f6GuXXHY25559KRUr+JZ9z712ZcSIF5g9e0611zp36sD48ZOWbk+YMJnOnTrkNFaptmQzrOtG4DTgeoCU0jsR8S/gglwGVuiiuJjidbqy4M5rKf/kQxofOphGex5MlDZl/r+vY8nrI2mwVR+aHP075l5+OuXjRjP7d4fCwgWUdN+aJif8kTlnHFXXt6E8+fDDMfz5z3/j0aH/Yt7cebz19nuUl1vSrS9SReL2AUNo1KIJ+9xwEm036kJxwwYsWbiYO/c6lw1370n/ywdx9wHnU1RcRPvN1uPeQy6mQeMGHPK/PzDpjTHM+HRyXd+G8sTfFwLov/vOTJv2JW+99S69d9im2usHHLg3t916dx1EplphQ3xG2QzrapJSeuVb+2rs0I6IQRHxWkS8dstHE1Y9ugJW8dU00oxplH/yIQCLX32W4nW60nD73Vjy+siqfSMoXr9b5QkL5sHCBQAseecVoriEaLb8WPI0p4xo0gyKKv+zFLVe4zsPC1Ph+uctd7FNrwHs3Hd/Zs6cxejRnyz3+oSJk+my1jffjq7ZpRMTJvqBdHWysGweX7z4Puvt1J05k75i9GOvATDmsddot/FaAMyZPINxz77DkvkLmT9jDuNf/pB2m6y93HXmz5hD4xZNiOLK3xXNOrVhzuQZ+b0Z5ZS/L9Sr15YM2KMv77w3gv+75a/s2GdbbrjpCgDatG3Nllt25/HHnl7huRMnTaFLl05Lt9dcsyMTJ03JS9zS95VNcjI9IjYAEkBEHABMqumElNINKaWeKaWeR3VbsxbCLDypbAYVX02jqGMXAEo22YKKieOomDmd4m6bA1D8o59QMaUyOYsW3wy5KF6vG0QRaU71seTlH75Fg547AtBg+91Y8sYLub4V5Um7dpVD9NZaqzP77TeAf9/13+Vef/jhJzjisAMA2GbrLSibVcbkyVPzHqdqV2mb5jRq0QSAkkYNWGeHzfhq7ETGPPE6a2/7I6Bydq6vKyNjnnidzlt1I4qLKGnckE4/2YAvR0+sdt3PX3yfjfbYGoBND9iBMU+8kac7Uj74+0J//MPlbNKtN9037cMvjjqRZ0e8yKBfVk6ws99+u/PYY0+zcOGiFZ771JPPsssuvWnVqgWtWrVgl11689STz+YzfK1MRUXdPgpYNsO6jgNuADaOiAnAp8BhOY3qB2L+HddSOuhMoqQBFdMmMe/mP1P85guUHjoYiopJixcx75arAGiw1Y403HlvKC+v3P+Pb0bFNTn5Qub/80rSzC+Zf+9NNPn1EBr97GgqPh/DgpGP1tXtqZbde/eNtGnbmsWLl3DCCUOYNauMQb86AoAbbrydoY8OZ/fdd+GjD55n3vz5/PKXp9RxxKoNTdu3YsCVxxLFRURR8NHDL/PJ8LeY8OrH7PHXwWzxywEsnruAJ06v7E/7asxEPnvmHQY+cTGpooJRdz3Dlx+PB+Cnt/yOJ35/E3OnzGTkxXex57W/ZfvTDmTqe5/x7t3P1OFdqrb5+0I1+dkBe3HVFdcvt+8nP9mMXxxzCMf/9ixmzJjFZZdey9Mj/gfApZdcw4wZs+oiVOk7i1Q1U0zGAyKKU0rlEdEUKEopzf4ubzDr6H41v4HqpbZ3flDXIagAXdpx57oOQQXm95NXPGxF9VvTho3rOgQVqFlzxkZdx5CNBc/fWaefjxtvf1jB/jtlUzn5NCIeA+4GnspxPJIkSdLqrcCHVtWlbHpONgaepHJ416cRcW1EVJ/bUpIkSZK+h5VWTlJK84B7gHsiojXwV2AEUJzj2CRJkqTVTkrldR1CwcpqhfiI6BMRfwdeBxoDB+U0KkmSJEn1zkorJxHxGfAmldWT01JKc3MdlCRJkqT6J5uG+O4ppeoLckiSJEn67myIzyhjchIRp6eULgMujIhq052llE7IaWSSJEmS6pWaKidfL0TxWj4CkSRJkuqFZOUkk4zJSUrpoaqn81JK9y77WkQcmNOoJEmSJNU72czWdWaW+yRJkiRpldXUczIA2ANYMyKuXualFsCSXAcmSZIkrZZsiM+opp6TiVT2m+xD5fomX5sNnJzLoCRJkiTVPzX1nLwNvB0R/0opLc5jTJIkSZLqoWzWOVk3Ii4GNqFydXgAUkrr5ywqSZIkaXXlbF0ZZdMQ/0/gOir7THYGbgPuyGVQkiRJkuqfbConpSml4RERKaVxwB8i4nXg3BzHJkmSJK1+bIjPKJvkZGFEFAGjI+K3wASgWW7DkiRJklTfZDOs60SgCXACsCVwBDAwl0FJkiRJqn9WWjlJKb1a9XQOcHRuw5EkSZJWczbEZ7TS5CQiHgLSt3bPonINlOtTSgtyEZgkSZKk+iWbnpNPgHbAv6u2f07lQowbATdSOcxLkiRJUjZsiM8om+Rku5TSVstsPxQRr6aUtoqI93IVmCRJkqT6JZuG+GYRsfbXG1XPv56ta1FOopIkSZJU72RTOTkVeC4ixgIBrAcMjoimwK25DE6SJEla7TisK6NsZusaGhFdgY2rdn20TBP8X3IWmSRJkqR6JZvZupoApwDrpJR+FRFdI6JbSunh3IcnSZIkrWacSjijbHpO/kllb8m2VdsTgAtyFpEkSZKkeimb5GSDlNJlwGKAlNI8KntPJEmSJKnWZNMQvygiSqlaiDEiNgAW5jQqSZIkaXVlQ3xG2SQn5wGPAWtFxJ3A9sBRuQxKkiRJUv2TzWxdwyLiDaAXlcO5TkwpTc95ZJIkSdLqyIb4jDImJ8suvFhlVNXPJhGxdkrp89yFJUmSJKm+qaly8giVfSbLNr8noB3QHijOYVySJEmS6pmMyUlKabNltyNiXeD3QD/gopxGJUmSJK2ubIjPaKVTCVctungL8CjwOrBJSumaXAcmSZIkqX6pqefkx8AQYFPgMuCYlFJ5vgKTJEmSVks2xGdUU8/J28AXVPaebA1sHfFN+0lK6YTchiZJkiSpPqkpOflF3qKQJEmSVO/V1BB/az4DkSRJkuoFG+IzWmlDvCRJkiTlg8mJJEmSpIJQU88JABHRNqX0ZT6CkSRJklZ7DuvKKJvKyUsRcW9E7BHLTtclSZIkSbUom+RkI+AG4AhgdERcFBEb5TYsSZIkaTWVUt0+CthKk5NUaVhK6RDgV8BA4JWIGBER2+Y8QkmSJEn1QlY9J8DhVFZOpgDHAw8CPYB7gfVyGaAkSZKk+mGlyQnwInA7sF9Kafwy+1+LiH/kJixJkiRpNWVDfEbZJCfdUlrx4LSU0qW1HI8kSZKkeipjchIRDwGp6nm111NK++QuLEmSJGk1ZeUko5oqJ5fnLQpJkiRJ9V7G5CSlNOLr5xFRCqydUvooL1FJkiRJqndWOpVwROwNvAU8VrXdIyIezHVgkiRJ0mopVdTto4BlswjjH4CtgZkAKaW3cPpgSZIkSbUsm9m6FqeUZn2rKb6wl5aUJEmSCpUN8Rllk5y8FxGHAsUR0RU4AXght2FJkiRJqm+yGdZ1PLApsBD4N1AGnJTLoCRJkiTVPyutnKSU5gFDqh6SJEmSvo8Vr28uskhOIqIncBaw7rLHp5S65y4sSZIkSfVNNj0ndwKnAaMAu3ckSZKk78OG+IyySU6mpZRc10SSJElSTmWTnJwXETcBw6lsigcgpXR/Nm/Q8a7RqxiaVmczBm9R1yGoAG1/19i6DkEFZpM2azO2bFJdh6ECc1mrbes6BEk5kk1ycjSwMdCAb4Z1JSCr5ESSpFVlYiJpteSwroyySU62Sil1y3kkkiRJkuq1bJKTFyJik5TS+zmPRpIkSVrdJSsnmWSTnPQC3oqIT6nsOQkgOZWwJEmSpNqUTXKye86jkCRJklTvZbNC/LiI6A10TSn9MyLaAc1yH5okSZK0+kkVrhCfSdHKDoiI84DfA2dW7WoA3JHLoCRJkiTVPytNToCfAvsAcwFSShOB5rkMSpIkSVL9k03PyaKUUoqIBBARTXMckyRJkrT6cp2TjLKpnNwTEdcDrSLiV8CTwI25DUuSJElSXYmIkyPivYh4NyL+HRGNI2K9iHg5IsZExN0R0bDq2EZV22OqXl93Vd+3xuQkIgK4G/gPcB/QDTg3pXTNqr6hJEmSVK+lirp9rERErAmcAPRMKf0YKAYOBi4FrkopbQjMAI6pOuUYYEbV/quqjlslNQ7rqhrONTSltBkwbFXfRJIkSdIPSglQGhGLgSbAJGAX4NCq128F/gBcB+xb9RwqixrXRkSklL7ztGTZDOt6IyK2+q4XliRJklR4ImJQRLy2zGPQsq+nlCYAlwOfU5mUzAJeB2amlJZUHTYeWLPq+ZrAF1XnLqk6vu2qxJZNQ/w2wGERMY7KGbtcIV6SJElaVXW8zklK6QbghkyvR0RrKqsh6wEzgXvJ08Ls2SQn/XMehSRJkqRC0Q/4NKU0DSAi7ge2p3KCrJKq6kgXYELV8ROAtYDxEVECtAS+XJU3zmZY1wUppXHLPoALVuXNJEmSpHqvoqJuHyv3OdArIppUTZDVF3gfeBo4oOqYgcADVc8frNqm6vWnVqXfBLJLTjZddiMiioEtV+XNJEmSJBW2lNLLVDa2vwGMojJnuAH4PXBKRIyhsqfk5qpTbgbaVu0/BThjVd8747CuiDgTOIvKLv2yr3cDi3CdE0mSJGm1lVI6DzjvW7s/AbZewbELgANr430zJicppYuBiyPi4pTSmbXxZpIkSVK95wrxGWUzrGvMshsRURwR386iJEmSJOl7ySY56RsRQyOiU0T8GHgJaJ7juCRJkqTVU0p1+yhgK51KOKV0aET8nMpmmLnAoSml53MemSRJkqR6ZaWVk4joCpwI3AeMA46IiCa5DkySJElS/ZLNIowPAcellIZXzXN8CvAq35piWJIkSVIWbIjPKJvkZOuUUhlA1WIqV0TEQ7kNS5IkSVJ9k3FYV0ScDpBSKouIb89bfFQug5IkSZJWWxWpbh8FrKaek4OXef7tdU52z0EskiRJkuqxmpKTyPB8RduSJEmS9L3U1HOSMjxf0bYkSZKkbCQb4jOpKTnZPCLKqKySlFY9p2q7cc4jkyRJklSvZExOUkrF+QxEkiRJUv2WzVTCkiRJkmpLgc+YVZdWukK8JEmSJOWDlRNJkiQpj5IrxGdk5USSJElSQTA5kSRJklQQHNYlSZIk5ZMN8RlZOZEkSZJUEKycSJIkSfnkCvEZWTmRJEmSVBBMTiRJkiQVBId1SZIkSflkQ3xGVk4kSZIkFQQrJ5IkSVI+uUJ8RlZOJEmSJBUEkxNJkiRJBcFhXZIkSVI+2RCfkZUTSZIkSQXByokkSZKUT64Qn5GVE0mSJEkFweREkiRJUkFwWJckSZKUTzbEZ2TlRJIkSVJBsHIiSZIk5VFyhfiMrJxIkiRJKggmJ5IkSZIKgsO6JEmSpHyyIT4jKyeSJEmSCoLJiSRJkqSC4LAuSZIkKZ8c1pWRlRNJkiRJBcHKSS1o1KgRTz55Dw0bNqSkpIT//ncoF1xwFb/+9UB++9tfsMEG69KlSw++/HLGCs8/7LD9OeOM4wG45JJruPPO+/IZvmpRtF+T0oGnL90uatuRhY/eSZr5JQ13P5SiDl2Yd9WpVHwxZvnzWrWj6Zl/Y+Fj/2bx0/+tft02HSgdeBrRpDnl48ey4I4roXxJzu9HteOPVw2hz67b8dX0Gfxsp8MBOO70Qey8+w5UVFTw1fQZnHPiBUybMh2A319wMjv03Y4F8xdwzonn88Goj6td80fdu3HBX8+hUeNGjBz+ApeefVVe70m1q1GjRjwx7G4aNWxEcUkx//vfo1x4wTf/Tf98+XkceeRBdGi/6QrP/93vBnPkwIMoLy/ntN/9kSeffDZfoasWFTdqwD73nU1xwxKiuJhPh77Ca1fcT+ftNqHXOYdS3KCYaaM+Y8TvbiSVV9Bp2x/R/+aTmf3FNAA+ffRV3vjL/6pdt/la7ej79+No3Lo509/5lKdOvI6KxeX5vj0tK7nOSSZWTmrBwoUL2X33Q9hmmwFss80AdtutD1tv/RNefPE19tjjMMaN+yLjua1bt2TIkJPYccd92WGHfRgy5CRatWqRx+hVm9LUCcz784mVj8tPJi1ayJJ3XqRi8jjm//Miyj95b4XnNdrvGJZ88HrG6zba+ygWPfMAcy88ljRvDg167ZqrW1AOPHj3I/zmkJOX23fL3+/ggF2O4KB+A3l22PMce8ovAOjdd1vWWX8t9tr2QP70u0s4+9LTV3RJzr70dP546sXste2BrLP+WvTepVfO70O5s3DhQvYYcCi9eg1g2157sOuufdhqq58A8JMtNqN1q5YZz9144w054IC96bnlbuy370Cu+sv5FBX55/2HqHzhYh466CL+s9sQ7us/hC47dafDll3Z+S/HMnzwtdzb70zmTJjORgfusPScya98xH39K49fUWICsM1ZBzPqxse4q/epLJw1l40P3ilPdyR9dyv97RWVDo+Ic6u2146IrXMf2g/L3LnzAGjQoISSkgaklHj77ff4/PPxNZ636659GD58JDNmzGLmzDKGDx/JbrvtlIeIlWvFG21Omj6JNGMaFVPGk6ZOWOFxJZv1In01hYrJn2e+VtfuLHn7eQAWvzqcks38IPpD8vpLbzFrZtly++bOmbf0eWmTUqBy/PHO/XfkoXseBeCdN96jeYtmrNG+7XLnrtG+Lc2aNeWdNyqT3YfueZSdd++TwztQPiz7d6RBgxISiaKiIi688CzOPvvijOfttddu/Oc/D7Fo0SLGjRvPJ2PH0bNnj3yFrVq2ZN5CAIpKiikqKSFVVFC+aAmzPp0MwPhn32X9Pbb6TtfsvP0mfPLIKwB8fO9I1u2/Ze0GLdWibL5a+TuwLXBI1fZs4G85i+gHqqioiJdeGsrnn7/BU0+N5NVX38rqvM6dOzJ+/KSl2xMmTKZz5465ClN51GCLHVj8xkqGVjRsTMO++7PwsX9nPCSatoD5c6CisgScZn5JtGyb8Xj9cBx/xrE88fr/2HP/3fjbZTcC0L5TOyZPnLL0mCmTptG+U7vlzmvfqR1TJk1d5pip1Y7RD09RUREvvjSUz8a9zlPDn+O1V9/i178eyNBHnmTy5GkZz+vUuQPjx09cuj1h4iQ6d+6Qj5CVA1EU7P/4hRz59t+ZMHIUU98cS1FJMWt0Xw+A9ffcmqadv/kb0GHLDTngiQsZcPtptN5ozWrXa9y6GYvK5pHKK/+GzJn0FU07ts7PzSizilS3jwKWTXKyTUrpOGABQEppBtCwphMiYlBEvBYRry1ZMqcWwix8FRUV9Oq1Bxtu2IuePXuwySYb1XVIqkvFJRRvug1L3nq+xsMa7X4oi555ABYtyFNgKiTXXHI9u225H4/c9wSH/OKAug5HdayiooJte+3BRl23Zcuem7P99lvz05/twXXX3VLXoSmPUkXivv5DuGOrE2jXYwNad+vC8MHXst15h/PTh//I4jnzlyYa00d9xp3bnMR/dhvCu/98gv43n7ySq0uFL5vkZHFEFFM15iAi2gE1dvGklG5IKfVMKfUsKWlWC2H+cMyaVcaIES9kPTRr4sTJdOnSaen2mmt2ZOLEyTmKTvlS8qMtqRg/ljRnZo3HFa2zEY32OYqm595Ewz770KjfgTTovedyx6S5ZVDaDKrGkEertqRZX+YsduXfI/c/Tr89dwJg6qRpdFzmW+8OndoxddLy35pPnTSNDp3aL3NM+2rH6Idr1qwynn32RXbssy0bbLAuo94dwfsfPEeTJqW8M+qZasdPmjiFLl06L91es3MnJi5TfdMP06KyeUx84X3W2qk7U94Yw4P7n89/9zqPSS9/yKxPKj8nLJ4zf+kwsC+eepuikmIat17+c9eCGXNo2KIJUVz5N6RZpzbMnbziCXqUP6ki1emjkGWTnFwN/BdoHxEXAs8BF+U0qh+YNdZoQ8uWlU3sjRs3om/fHfjoozErOavSsGEj6NdvR1q1akGrVi3o129Hhg0bkctwlQclW+zI4jdW/t9x/jVnMPdPv2Tun37JohEPsvDJe1n83CPVjisf8w4lm28PQIOt+rJk1Mu1HrPya+31uix9vvPuO/DpmHEAPPPESPY+aAAA3STH9j0AACAASURBVLfYlNmz5zJ96vLJ6PSpXzJnzly6b1E5c9PeBw3g6cednemH7Nt/R3bZpTdvvjmK9dfbik1+1JtNftSbefPm032znaqd+8gjwzjggL1p2LAh66zThQ02XJfXXstuaLEKS+M2zWnYogkAxY0b0GWHzZg5ZiKN21b+b6OoYQk9Bu/N+7cPB6C03TcTJbTrsT4UBQtmVB+xMvGF91l/z8p24Y0O3IHPnngj17cirbKVTiWcUrozIl4H+gIB7JdS+iDnkf2AdOzYnhtvvJLi4iKKioq4776HefTRpxg8+ChOOeXXdOjQjldffZzHHnuawYN/zxZbbMYvf3k4gwf/nhkzZnHxxVfz3HMPAXDRRX9lxoxZdXxH+l4aNqKkWw8W3PNNa1bJZr1otP+xRLOWlA46l4oJnzL/H+fVeJnSQeex4K5rSGVfsfChWyg98nQa7XE45RM+YfFLT+T6LlSLLr3uj/TcbgtatWnFsDce4O9/vokd+m7LuhuuTUVFYtL4yZx/+mUAjHzyBXboux2PvHQvC+Yv5JyTLlh6nXuevJWD+g0E4MIz/swFfz2bRo0b8dxTL/Hc8Bfr5N5UOzp2bM8NN15BcVHV35H7H+GxR5/KePwee/Zjiy0244Lzr+KDD0Zz3/0P8/obw1iyZAmnnHwuFRVOU/pD1KRDK3a+6liiuIiIYOzDL/P58LfodfYhrN23B1FUxPu3PcnEF94HKvtPNjmiL6m8nCULFjN88Dd/dwbc9jtGnHYT86bM5OWL7qLf33/LVqcfyPR3P+PDu56pozuUVi5Sqrm0ExFXA3ellF5YlTcoLV2nsGtHqhNTj+1e1yGoAG1/1/S6DkEFZmzZpJUfpHrnyja96zoEFahjx98RdR1DNmafsFedfj5ufvXDBfvvlM2wrteBsyNibERcHhE9cx2UJEmSpPonm2FdtwK3RkQbYH/g0ohYO6XUNefRSZIkSasbh15m9F2WkN0Q2BhYB/gwN+FIkiRJqq+yWSH+sogYDfwJeBfomVLaO+eRSZIkSapXVjqsCxgLbJtSslNVkiRJ+r4KfK2RupQxOYmIjVNKHwKvAmtHxNrLvp5ScpJsSZIkSbWmpsrJKcAg4IoVvJaAXXISkSRJkrQ6s3KSUcbkJKU0qOrpgJTSgmVfi4jGOY1KkiRJUr2TzWxdK1p8cZUWZJQkSZKkTGrqOekIrAmURsRPgK9XkmwBNMlDbJIkSdJqJyWHdWVSU89Jf+AooAtw5TL7ZwNn5TAmSZIkSfVQTT0nX68Mv39K6b48xiRJkiStvmyIz6imYV2Hp5TuANaNiFO+/XpK6coVnCZJkiRJq6SmYV1Nq342y0cgkiRJkuq3moZ1XV/184/5C0eSJElazTmsK6OVTiUcEZdFRIuIaBARwyNiWkQcno/gJEmSJNUf2axzsltKqQzYC/gM2BA4LZdBSZIkSap/auo5+fYxewL3ppRmRURNx0uSJEnKIDmsK6NskpOHI+JDYD7wm4hoByzIbViSJEmS6puVJicppTMi4jJgVkqpPCLmAvvmPjRJkiRpNWTlJKOVJicR0QA4HNixajjXCOAfOY5LkiRJUj2TzbCu64AGwN+rto+o2vfLXAUlSZIkqf7JJjnZKqW0+TLbT0XE27kKSJIkSVqtVdR1AIUrm6mEyyNig683ImJ9oDx3IUmSJEmqj7KpnJwGPB0RnwABrAMcndOoJEmSpNWUUwlnVmNyUjVt8Cxga6B91e6PUkoLcx2YJEmSpPol47CuiPgl8B5wDfAWsG5K6R0TE0mSJEm5UFPl5CRg05TStKo+kzuBB/MTliRJkrSaclhXRjU1xC9KKU0DSCl9AjTKT0iSJEmS6qOaKiddIuLqTNsppRNyF5YkSZK0mnIq4YxqSk5O+9b267kMRJIkSVL9ljE5SSndms9AJEmSJNVv2axzIkmSJKmWuM5JZtmsEC9JkiRJObfSyklEtE0pfZmPYCRJkqTVng3xGWVTOXkpIu6NiD0iInIekSRJkqR6KZvkZCPgBuAIYHREXBQRG+U2LEmSJEn1zUqHdaWUEjAMGBYROwN3AIMj4m3gjJTSizmOUZIkSVpt2BCfWVY9J8DhVFZOpgDHAw8CPYB7gfVyGaAkSZKk+iGbqYRfBG4H9kspjV9m/2sR8Y/chCVJkiSpvskmOelWNbSrmpTSpbUcjyRJkrR6c7aujDImJxHxEJCqnld7PaW0T+7CkiRJklTf1FQ5uTxvUUiSJEn1RLJyklHG5CSlNOLr5xFRCqydUvooL1FJkiRJqndWus5JROwNvAU8VrXdIyIezHVgkiRJkuqXbBri/wBsDTwDkFJ6KyKcPliSJElaFQ7ryiibFeIXp5RmfWufK8dIkiRJqlXZVE7ei4hDgeKI6AqcALyQ27AkSZKk1ZMN8ZllUzk5HtgUWAj8GygDTsplUJIkSZLqn5VWTlJK84AhVQ9JkiRJyomVJicR0RM4C1h32eNTSt1zF5YkSZK0mnJYV0bZ9JzcCZwGjMJ/SkmSJEk5kk1yMi2l5LomkiRJUi2wIT6zbJKT8yLiJmA4lU3xAKSU7s9ZVJIkSZLqnWySk6OBjYEGfDOsKwEmJ5IkSZJqTTbJyVYppW45j0SSJEmqBxzWlVk265y8EBGb5DwSSZIkSfVaNpWTXsBbEfEplT0nASSnEpYkSZK+OysnmWWTnOye8ygkSZIk1XsrHdaVUhoHrAXsUvV8XjbnSZIkSdJ3kc0K8ecBPYFuwD+pnLXrDmD7bN7g6rY7fJ/4tJra9La36zoEFaCxHz9Q1yGowOz5k8F1HYIK0NSiVNchSN9PirqOoGBlUwH5KbAPMBcgpTQRaJ7LoCRJkiTVP9n0nCxKKaWISAAR0TTHMUmSJEmrLRviM8umcnJPRFwPtIqIXwFPAjfmNixJkiRJ9U2NlZOICOBuKleIL6Oy7+TclNKwPMQmSZIkqR6pMTmpGs41NKW0GWBCIkmSJH1PqcKG+EyyGdb1RkRslfNIJEmSJNVr2TTEbwMcFhHjqJyxyxXiJUmSJNW6bJKT/jmPQpIkSaonfgizdUVEK+Am4MdAAn4BfERlP/q6wGfAQSmlGVV96n8F9qBywfajUkpvrMr7ZjOs64KU0rhlH8AFq/JmkiRJkn4Q/go8llLaGNgc+AA4AxieUuoKDK/aBhgAdK16DAKuW9U3zaZysumyGxFRDGy5qm8oSZIk1WepwFeIj4iWwI7AUQAppUXAoojYF9ip6rBbgWeA3wP7ArellBLwUkS0iohOKaVJ3/W9M1ZOIuLMiJgNdI+IsqrHbGAq8OB3fSNJkiRJdS8iBkXEa8s8Bn3rkPWAacA/I+LNiLipaiH2DsskHJOBDlXP1wS+WOb88VX7vrOMlZOU0sXAxRFxcUrpzFW5uCRJkqTCklK6AbihhkNKgC2A41NKL0fEX/lmCNfX10gRkWo7tmx6TsYsuxERxRFxXm0HIkmSJNUHqaJuH1kYD4xPKb1ctf0fKpOVKRHRCaDq59Sq1ycAay1zfpeqfd9ZNslJ34gYGhGdIuLHwEtA81V5M0mSJEmFLaU0GfgiIrpV7eoLvE9la8fAqn0DgQeqnj8IHBmVegGzVqXfBLJoiE8pHRoRPwdGUbnOyaEppedX5c0kSZKk+u4HskL88cCdEdEQ+AQ4msrCxj0RcQwwDjio6tihVE4jPIbKqYSPXtU3XWlyEhFdgROB+4AfAUdExJsppXmr+qaSJEmSCldK6S2g5wpe6ruCYxNwXG28bzbDuh4CzkkpHQv0AUYDr9bGm0uSJEnS17JZ52TrlFIZLM2KroiIh3IbliRJkrR6SrU+x9Xqo6Z1Tk4HSCmVRcSB33r5qFwGJUmSJKn+qWlY18HLPP/2Oie75yAWSZIkabWXKqJOH4WspuQkMjxf0bYkSZIkfS81JScpw/MVbUuSJEnS91JTQ/zmEVFGZZWktOo5VduNcx6ZJEmStBoq9KFVdSljcpJSKs5nIJIkSZLqt2ymEpYkSZJUS5xKOLNsFmGUJEmSpJwzOZEkSZJUEBzWJUmSJOWRDfGZWTmRJEmSVBCsnEiSJEl5lJKVk0ysnEiSJEkqCCYnkiRJkgqCw7okSZKkPEoVdR1B4bJyIkmSJKkgmJxIkiRJKggO65IkSZLyqMLZujKyciJJkiSpIFg5kSRJkvLIdU4ys3IiSZIkqSCYnEiSJEkqCA7rkiRJkvIoVTisKxMrJ5IkSZIKgpUTSZIkKY9SqusICpeVE0mSJEkFweREkiRJUkFwWJckSZKURzbEZ2blRJIkSVJBsHIiSZIk5VGFK8RnZOVEkiRJUkEwOZEkSZJUEBzWJUmSJOVRclhXRlZOJEmSJBUEKyeSJElSHrlCfGZWTiRJkiQVBJMTSZIkSQXBYV2SJElSHrnOSWZWTiRJkiQVBCsnkiRJUh45lXBmVk4kSZIkFQSTE0mSJEkFwWFdq6i4UQP2uu9sihuWUFRczCdDX+GNK+6n83absM05h1LUoJjpoz7j2d/dSCqvoEHzUna++jc0W7MtRcXFvHP9UD6+59lq111js3Xpc9WxFDduyBdPvcWL595eB3en2nLMb47gkCN+RkqJD98fze9+ew4XXXEu22y/JbPL5gBw6nFn8/67H1U794CD9+H4UwcBcM0VN/Cfux7Ma+z6fs6+6Eqeff4V2rRuxf/u+AcAs8pmc+o5FzNx8hQ6d+zAFeefScsWzXn48ae4+c57IUGTJqWc87vfsnHX9Zk0ZRpnnX85X86YQRAcsO8Ajjhov2rvlVLi4r/8g5Evvkrjxo24cMipbNJtw3zfsr6jUy4/mV59t2HmlzMZ1O/XAKz/o/U44eITKG3amClfTOGSEy5j3px5dOuxESddcmLliRHccdUdPP/YC9Wu2XGtDpz1tzNp3roFo0eN5rIT/8ySxUvyeVv6HoobNWDgPedQ0rCEopJiPhj6CiOuum/p6/3/cCQ9DurDpZscA8AWh/VlqyN3paK8gkXzFvDImTczffSEatfdoE93+p93BFFcxJt3PcML1z2Ut3vSirnOSWZWTlZR+cLFPHLQRdy/2xDu6z+EtXbqTvstu9LnL8fy1OBrua/fmcyZMJ2NDtwBgE0H7srM0RO4f7chPHzghWxzbmUC823bX3w0I0+/iXt6n0rL9TrSZefu+b411ZIOndpz9KBD2XOXg9l1+59RXFzM3j8bAMBF513JgD4HMqDPgStMTFq2asFJp/+GfXY9lH36HcpJp/+Gli1b5PsW9D3st8eu/OPKC5bbd9Pt99CrZw+G3n0zvXr24OY77gFgzc4dueXay/jv7dfx66MO4Y+XXQ1ASXExpx3/Kx688wb+dcNV3HX/w4z9dFy19xr54qt8Pn4iQ+++mT+cfgLnX35t7m9Q39uwe4dx1hFnL7fv5D+fzM2X/B/H7vobnn/8BQ789QEAfPbhOI7b83h+s/txDDnibE68+ASKiqv/CT/mzGO4/6b/cvQOv2DOzDnsfnD/vNyLakf5wsXcfsiF3DDgLG4YcBYb9OnOmj+p/KKh02br0bhl0+WOf/eBF7i+/xncuMdZvPiPh9n17MOqXTOKgt3PP4p/DbyM6/qdzo/32ZY1uq6Zj9uRVonJyfewZN5CAIpKiikqKSFVVFCxaAmzPp0MwIRn32XdPbYCKr/ZbNC0FIAGTRuzcOZcKpZULHe90vataNislKlvjAVg9H+eY93+PfN1O8qBkpISGjduRHFxMaWljZkyeWpW5/XZZXtGPvMis2aWMWtWGSOfeZE+fbfPcbSqTT17bEbLFs2X2/f0yBfZd0A/APYd0I+nnn0RgJ9stsnSY7tvujFTpk4HoN0abZZWQJo2bcL666zFlGlfVnuvp597iX1270tEsPmPf8Ts2XOYNv2rnN2baseol99l9szZy+3rst6ajHppFABvPPsGvQdU/v9+4YKFVJRX/s1o2KgBKcPXrj2235xnHxkJwLD/PMl2/bfLVfjKkcXLfrZoUExKiSgK+g05lOEX/3u5YxfNmb/0eYMmjVZ4vc49NmDGZ1OY+cU0KhaX895DL9Ft1y1zdwPS95R1chIR60REv6rnpRHRfGXnrO6iKPjZ4xdyxNt/Z8LIUUx7cyxRUswa3dcDYL09t6ZZ57YAvH/LMFp17cxhr1/L/k9eXDlc61t/XJp2bM3cSd98oJg76SuadmydvxtSrZoyaSo3XHsLL70zjNc+eIqysjmMfLryw+hpQ47n8ZH3ce6Fp9OwYYNq53bs3J6JEyYv3Z40cQodO7fPW+zKjS9nzKTdGm0AWKNta76cMbPaMfc//Di9e1X/UmLCpCl8MHos3TftVu21KdO+pGP7NZZud2i/BlOmTa/FyJUvn308ju36bwvAjnvtSLvO7Za+tnGPbtzw5PVcP+wfXH3WNUuTla+1aN2COWVzl+6fPmkaa3Rsm7/gVSuiKPjV0Is49Y3r+HTku0x8ayxbDdyNj4e9zpyp1X9n9DxyV4579kr6nnkIj593a7XXW3RsQ9mkb77UKJv0Fc39bFHnKlLU6aOQZZWcRMSvgP8A11ft6gL8L1dB/VCkisT9/Yfwr61OoF2PDWjdrQtPDb6Wbc87nH0f/iOL58wnVf2R6LLTZnz53jju3PK33N9/CNtfcCQNmpXW8R0ol1q2bMGuA3Zm+5/szlab9KVJk1J+euBeXHr+X9h5m33Yu+/BtGrVgt+ceExdh6o6EBFELP8H4pXX3+b+h5/glMG/WG7/vHnzOXnIBfz+hGNp1nT5YR1avVz5uyvZ+8i9+Nsj11DatHS5fpEP3/qIQf2O5bd7ncDPj/s5DRpV/2JDP3ypInHjHmfxl17H07nHBqy99cb8aM9teOWWJ1Z4/Gu3DeNvO57CU5fcRe/jq/ekST802VZOjgO2B8oAUkqjgYxf40bEoIh4LSJee3bu6O8fZYFbVDaPiS+8T5edujP1jTE8tP/5PLDXeUx6+UNmfVL57fdGB/Xhs0dfA6DssynM/mIarTbstNx15k6eQdNObZZuN+3UhrmTZ+TvRlSreu/Uiy8+n8BXX85gyZIlPPbwk2y59eZMnVL5jfaiRYu551//o8cWP6527uSJU+m8Zsel2506d2DyxOyGhKlwtW3daulwq2nTv6JNq5ZLX/tozKece8lfuOaSc2m1TH/R4iVLOGnIBey5287sutOKh/Z1aNeWyVO/qZRMmTqdDu3WWOGxKmxfjB3PmYcN4bg9j+fpB55h4rhJ1Y8Z8wUL5s5n3W7rLre/bEYZzVo0XdqLskandkyfXH0YoH4YFpbN47MX3mfdbTehzTod+O2IKzn+ub/QoLQhx424otrx7z74It12q151LZv8FS06fVNBa9GpDbP9bFHnUoo6fRSybJOThSmlRV9vREQJkHGegZTSDSmlnimlnjs27fp9YyxIjds0p2GLJgAUN25Alx02Y9aYiTRuW/mhoqhhCZsP3psPbh8OwJwJ0+nce1MAStdoQcsNOlE2bvkPm/OnzmTRnPm032IDALoe0JtxT7yer1tSLZswfhJb9OxO49LGAGy/4zaM+fhT2nf45kNj/z134aMPxlQ7d8RTz7PDztvSsmULWrZswQ47b8uIp57PW+zKjZ169+KBR58E4IFHn2TnHSqH70yaPJWTzjqfi889jXXX7rL0+JQS5178F9ZfZy0GHvyzGq/74GPDSSnx9rsf0KxZ06XDx/TD0qptZcIaERx6wiE8cscjQOUsXF8nHe3XbM9aG67FlC+mVDv/7RfeYcc9Kydi2fWAfrz4xIt5ily1oUmb5jSq+mxR0qgB6+/wYyaN+pSrtjqOa3qfxDW9T2Lx/EX8rc+pALRZt8PSc7vu0oOvPptc7ZoT3/6ENut1pNVa7ShqUMyme/fi42F+tlDhynYq4RERcRZQGhG7AoOBej0PXZMOrehz1bFEcRERwScPv8znw99i67MPYe2+PYiiIj647UkmvvA+AG/+9X/0ufJY9n/yYgBeuehuFs6onEr2Z49fyP39hwDw/Fm30OfKQZQ0bsgXz7zNF0+9XTc3qO/trddHMfTBYQx9+h7Ky5fw3jsf8q9b7+XWe66j7RptiID3Rn3EWaf+CYDuPTbhsKMP4vcn/oFZM8u4+vLreWh4ZfPjX/98PbNmltXl7eg7Ou28S3j1zXeYObOMvvsdzuBjjuCXRxzEqedcxP0PP07nju254vyzALjun/9iVtlsLrj8bwAUFxdzz/9dzZvvvMdDjw2n6wbrsv/A4wA48diB7Ljd1tz938oPrT//6Z7suO1WjHzxVQYc9AtKGzfm/LNOrpub1ndy5rVn0L1Xd1q2acGdr9zO7VfcQeOmjdln4N4APPfo8zx+d+VQnk23+jF/GnwQ5UuWUFGRuGbItZTNqPydcMGtf+LK0//CV1O+4qaLb+asv53JwNMGMvbdsTx21+N1dn/67pq1b8W+V/6aKCoiioL3H36Z0U+9mfH4ngN3Y/3eP6Z8cTkLyuby4Cn/WHqdvS77FXcd9WdSeQWPnXsLh972e6K4iLfvGcG0FUw3LBWKyDTjx3IHRRQBxwC7AQE8DtyUsjj5xi6HO5Ozqjl/nkmXqhv78QN1HYIKzJ4/GVzXIagA7VBko79W7Jxxdxb2mKUqL3f+WZ1+Pt5m4v0F+++UbeVkP+C2lNKNuQxGkiRJUv2Vbc/J3sDHEXF7ROxV1XMiSZIk6TtKdfwoZFklJymlo4ENgXuBQ4CxEXFTLgOTJEmSVL9kXQFJKS2OiEepTLhKqRzq9ctcBSZJkiSpfskqOYmIAcDPgZ2AZ4CbgINyFpUkSZK0mir0VdrrUraVkyOBu4FjU0oLcxiPJEmSpHoqq+QkpXRIrgORJEmS6oNCX6W9LtWYnETEcyml3hExm+Wb+wNIKaUWOY1OkiRJUr1RY3KSUupd9bN5fsKRJEmSVF9lNZVwRNyezT5JkiRJNauo40chy3YRxk2X3ahahHHL2g9HkvT/7d13nFTV2cDx37O7wErvXUWssSDW2MUSa6LGmrwaNdHYS+yxa4wlGqNJjFESY0nsscRuDGLvIopgAcWCNEE6ImXP+8dccIEdWHBndtj9ffnMZ+89c8tzdw8z99znnHslSWqsljbm5GzgHGCliJg6vxiYDfQvcGySJElSg5NwQHw+S8ycpJQuz8abXJVSap29WqWUOqSUzi5SjJIkSZIagdreSvjsiGgHrAlUVit/rlCBSZIkSWpcavuE+COBk4GewGBgC+BlYMfChSZJkiQ1PFVp6cs0VrUdEH8ysBnwaUppB2AjYHLBopIkSZLU6NS2cTIrpTQLICKapZTeB9YuXFiSJEmSGptadesCRkVEW+BB4KmImAR8WriwJEmSpIapyrt15VXbAfE/ziYvioiBQBvgiYJFJUmSJKnRqe2A+PbVZodkPx3KI0mSJC0jn3OSX23HnAwCvgQ+BIZn059ExKCI8EnxkiRJkr6z2jZOngL2SCl1TCl1AHYHHgGOA64vVHCSJEmSGo/aNk62SCk9OX8mpfRfYMuU0itAs4JEJkmSJDVAVfX8KmW1vVvXmIg4C7grmz8IGBcR5ZT+MUqSJElaAdS2cfJ/wIXkbiWcgBezsnLgwMKEJkmSJDU8DojPr7a3Ep4AnBgRLVJKMxZ5e0TdhyVJkiSpsanVmJOI2CoihgHvZfMbRoQD4SVJkiTVmdoOiL8G2BWYCJBSehvYrlBBSZIkSQ2VA+Lzq23jhJTS54sUzavjWCRJkiQ1YrUdEP95RGwFpIhoApxM1sVLkiRJUu2VevaiPtU2c3IMcDzQA/gC6JvNS5IkSVKdWJa7dR1c4FgkSZIkNWJLbJxExAVLeDullC6p43gkSZKkBs3nnOS3tMzJos80AWgBHAF0AGycSJIkSaoTS2ycpJSunj8dEa3IDYT/OXAXcHW+9SRJkiTVrMrESV5LHXMSEe2BU8mNObkV2DilNKnQgUmSJElqXJY25uQqYF+gP7BBSml6UaKSJEmS1OgsLXNyGvANcB5wbsSCHFSQGxDfuoCxSZIkSQ1OlQPi81ramJNaP0FekiRJkr6L2j4hXpIkSVIdSPUdQAkzMyJJkiSpJNg4kSRJklQS7NYlSZIkFVFVfQdQwsycSJIkSSoJNk4kSZIklQS7dUmSJElFVBU+5yQfMyeSJEmSSoKZE0mSJKmIfM5JfmZOJEmSJJUEGyeSJEmSSoLduiRJkqQi8jkn+Zk5kSRJklQSzJxIkiRJRVTlnYTzMnMiSZIkqSTYOJEkSZJUEuzWJUmSJBVRFfbrysfMiSRJkqSSYOZEkiRJKiKfEJ+fmRNJkiRJJcHGiSRJkqSSUPBuXadNeqnQu9AKqKKsvL5DUAk6dJNT6zsElZj/XLhufYegEnTbRePrOwTpO/E5J/mZOZEkSZJUEhwQL0mSJBVRVX0HUMLMnEiSJEkqCTZOJEmSJJUEu3VJkiRJReRzTvIzcyJJkiSpJNg4kSRJkoqoKur3VVsRUR4Rb0XEI9n8ahHxakSMiIi7I6JpVt4smx+Rvd9reX83Nk4kSZIk1eRk4L1q878DrkkprQFMAo7Iyo8AJmXl12TLLRcbJ5IkSZIWEhE9gT2Bv2fzAewI/Dtb5FZgn2x672ye7P2dsuWXmY0TSZIkqYiq6vkVEUdFxBvVXkfVEOa1wJl8+1iWDsDklNLcbH4U0COb7gF8DpC9PyVbfpl5ty5JkiSpEUkp9Qf653s/In4IjE8pvRkR/YoWGDZOJEmSJC1sa2CviNgDqARaA38E2kZERZYd6Ql8kS3/BbAyMCoiKoA2wMTl2bHduiRJkqQiqu9uXUuTUjo7pdQzpdQL+AnwdErpYGAgsH+22GHAf7Lph7J5svefTikt1+NcbJxIkiRJqo2zgFMjYgS5MSU3ZeU3AR2y8lOBXy/vDuzWJUmSJBVRWq77WNWPlNIzwDPZ9MfAoXIRogAAIABJREFU5jUsMws4oC72Z+ZEkiRJUkmwcSJJkiSpJNitS5IkSSqi2gxKb6zMnEiSJEkqCWZOJEmSpCIyc5KfmRNJkiRJJcHGiSRJkqSSYLcuSZIkqYiW69HpjYSZE0mSJEklwcyJJEmSVERVK9AT4ovNzIkkSZKkkmDjRJIkSVJJsFuXJEmSVEQ+5yQ/MyeSJEmSSoKZE0mSJKmIzJzkZ+ZEkiRJUkmwcSJJkiSpJNitS5IkSSoinxCfn5kTSZIkSSXBxokkSZKkkmC3LkmSJKmIqqK+IyhdZk4kSZIklQQzJ5IkSVIR+ZyT/MycSJIkSSoJNk4kSZIklQS7dUmSJElF5HNO8jNzIkmSJKkkmDmRJEmSiqjK3EleZk4kSZIklQQbJ5IkSZJKgt26JEmSpCLyOSf5mTmRJEmSVBLMnEiSJElF5HD4/MycSJIkSSoJNk4kSZIklQS7dUmSJElF5ID4/MycSJIkSSoJZk4kSZKkIqqK+o6gdJk5kSRJklQSbJxIkiRJKgl265IkSZKKqMonneRl5kSSJElSSTBzIkmSJBWReZP8zJxIkiRJKgk2TiRJkiSVBLt1SZIkSUXkE+Lzs3FSh8rKynj2hf8wZvQ4Dtz/SK67/go22ngDIoIRw0dy7NFnMGPGzMXWO/X0Yzn00AOYN6+KM8+4mAH/e74eolchDH53INOnz2DevCrmzp3LTtvvu+C940/8BZdcdjZr9NqcryZOWmzdn/zfjzntjOMAuPqq67nrjgeKFrfqVvtuHTnumpNp07EtpMSAO/7LEzc/suD9PX+5N4ec93OO6vszpk2axtb7bMdex+wLEcya8TU3nXsDn733yWLb7bRyZ0768+m0bNeKkUM+4i+nXMu8OXOLeGT6Lv75xsc88M7nRMCaHVtz8e59eOCdz7n9zZF8PnkmA4//Ae2aNwVg4PCxXP/Ch0QEFWXBGTuuy0Y92y+2zWFjp3DB42/zzdx5bNO7M2fuuC4RPu1tRVHerAl73Xce5U0riPJyRj72Gm9cfT/dt1qXLc7/P8qblPPlkE949vS/keblTm+7bfk9trroEMoqypk1aRoP73/pYttttXIndrr+eCrbtWLCOyN5+uS/UjVnXrEPT6oVGyd16Njjf86HH3xEq1YtATj7rN8ybdp0AC674lyOOuZQrrn6hoXWWXudNdhv/x+y+aa70a1bZx565J9stOFOVFXZpm4o9trzZ4s1Pnr06MoOO27D5599UeM6bdu14cxfn8iO2/+YlBIDn3uQxx8bwJTJU4sRsupY1bx5/Ou3N/PJux9T2aKSyx65miEvDOaL4aNo360jG2zbly9HjV+w/PjPx/GbA89lxtQZbNhvY355+XGcv8+Zi233/359GI/d9BAvP/wCR1x6DDsctDP/+9cTxTw0Ladx02Zx56BPuP/n21PZpJwzHhrEE++Ppm+Pdmy7emeOvOuVhZb//qod6bdGFyKCD8dP5cyHB/HgEf0W2+6lTw3hgl03YINubTnhvtd5ceSXbNO7c5GOSt/VvG/m8PCBlzF35jeUVZSz1wPn8/kzQ9jh2qN55KDLmTJyLJuevh9rHbAtH9z1LE1bN2fbSw/nsUOuZProiVR2aF3jdr9/zk8Y8rcn+OihV9j28p+zzk/6MeyfA4p8dFLtOOakjnTv3pVdd9uBW2+5e0HZ/IYJQGVlJSktfm+GPX/4A+779yPMnj2bTz8dxccff8qmm25YlJhVfy694lwuPP/KGusEwI47bcszA19k8qQpTJk8lWcGvshOO29X5ChVVyaPn8Qn734MwKwZs/hixCjad+kAwKEX/II7Lr91oVu3DH/zA2ZMnQHAiEEf0L5bhxq3u95WG/DqYy8B8Nx9A9l0l+8X8ChU1+ZVJb6ZO4+5VVXMmjOPTi0qWadLG3q0ab7Yss2bVizIgHw9Zx415UK+nD6LGbPn0qd7OyKCH67Xg4HDxxX4KFTX5s78BoCyinLKKipIVVXMmz2XKSPHAjDquXfpvcdmAKyxz1aMfPx1po+eCMCsiTVfwOq+9bp8/OhrAHx47/P02nWTQh+GlqKKVK+vUrbEzElELJ4zrial9FXdhrPiuuLK87ng3Cto2arFQuXX33Alu+zaj/ffG865Zy+eau3erQuvv/7WgvkvvhhLt+5dCx6viiOlxH0P3kxKiVtvvotbb76b3ffciTGjxzH03ffzrte9exe+GDVmwfzoL8bSvXuXYoSsAuvYszO91uvNiMEfsskPNuersRNr7LI1X7+f7MzgZwYtVt6qXStmTJ1BVda1Y+KYibTvusSPbJWQLq0qOXSz3ux249NUVpSzRa+ObLVapyWu8/SHY/nT8+/z1czZ/HnfzRZ7f/z0WXRpWVltHysxfvqsOo9dhRVlwb6P/5Y2vbow9NanGP/WR5RVlNOxz2pMeGckvffcnBbdcxcs2vbuSllFOT+691yatKhkyE1PMvy+FxbaXmW7lsyeOnNBN7DpY76iRdd2RT8uqbaW1q3rTXLX82q6SJOA3jWtFBFHAUcBNGvagaYVNacZG4rddtuRCV9OZPDgd9lm24WvXB53zJmUlZXx+6svYt/9f8jt//x3PUWp+rDHLj9lzJhxdOzYnvsfuoUPP/yYU087ln33Oby+Q1M9aNa8klNuOIvbfnMT8+bOY5/j9+eyn12Ud/l1t1yfHQ7amYv2O6d4Qaoops6awzMjxvHoUTvQqlkTznhoEI8OHcWe6/XMu86Oa3Vlx7W68ubnE7n+hQ+48aAtihixiiVVJe7b9Vyatm7OLn//Fe3W7smA465jqwsPobxZBaOeHbKgoREVZXTssxqPHHQ5FZVN2Oehixg/aMSCLItKV2nnLurXErt1pZRWSyn1zn4u+qqxYZKt1z+ltGlKadOG3jAB+P6Wm7D7njsxZNhz3Hzrn9hu+y35201/WPB+VVUV//73w+y9926LrTt6zDh69Oy+YL5Hj66MGe2HSkMxZkyuS8WECV/x6MNPsfXWm7NKr548/9LDDH53IN17dOWZ5x+kc+eOC603evQ4evTstmC+e4+ujB5t94wVWXlFOafccBYvPvgsrz/xCl1W7UanlTvzu8ev5U8v9Kd9tw5c9ugfaNOpLQCrrLMqR/3uBH5/5OVMnzxtse1NmzSNFq1bUFae+xjv0K0DX401mb2ieOXTCfRosxLtmzejSXkZO63ZlcGjF78xRk02WbkDo6bMZNLM2QuVd25ZybhqmZJx076mc7VMilYss6fOZPRLw1i5Xx/GDRrBQ/tdwgM/vJAxr77PlI9z5wkzxkxi1LPvMPfrb5g1aTpjXn2fDuuustB2Zk2aTtPWzYnss6Jlt/bMGFu7uibVh1qPOYmIdhGxeURsN/9VyMBWJBdfeBXfW2trNlh3O35+2Ek89+zL/PKIU+nde9UFy+yx5858+OFHi6372KP/Y7/9f0jTpk1ZddWe9F69F2+88XYxw1eBNG++Ei1btlgwvcNO2/DWoHdYu/cW9F1/B/quvwOjvxhLv233Yfz4CQut+/SA59lhx61p07Y1bdq2Zocdt+bpAd7FbUV21JUnMHrEKB77+0MAfP7BpxyzyeGctM1RnLTNUXw1ZiLn7HkqU76cTIfuHTnlxl/zl1OuYezI0Xm3OfTlIXx/j60A2G6/HXjzqdeKciz67rq1quSd0ZP5es48Ukq8+tkEendomXf5zybNWDBG7b1xU5g9r4q2KzVZaJlOLStp0bSCd0ZPIqXEI0O/oN+adgddkVS2b0XT1rkxR+WVTei57QZMHjF6wUD3sqYV9D3uRwsGs3/y5Jt03WxtoryMisqmdO67OpNGLP6ZMfqlYfTec3MA1jpgWz757+JdRaVSUau7dUXEkcDJQE9gMLAF8DKwY+FCW7FFBDf0v4pWrVsRAe8OeZ9TTj4fgN332ImNN96AS397Le+/N5wH7nuU1998krlz53H6qRd6p64GolPnjvzzjr8AUFFRwb/veXiJt4nuu9H6/PyIn3LyCecyedIUfn/l9Qx45n4ArvrdX5g8aUpR4lbdW3vT77Hdfjvw2XufcPlj1wBw91X/YvDAN2tcft+TD6Jlu1b84pJjgNzdvs790ekAnHnL+fztzOuYNH4Sd15+GydedxoHnn4wnwz9mIF3P1WcA9J3tkH3duy8Vjd+etvzlJcF63Ruw359VuGON0dyy2sfM3HGNxx4y3Ns07szF+7WhwEfjuXhoaOoKCujsqKMK3+08YIB8gfe8jz3HL4tAOf8YP3crYTnVLF1705ss5RxLCotzbu0ZYdrjibKy4gIPnrkVT4bMJgtzvspq+zUlygrY9ht/2P0S8MAmDxiNJ8/8w4HPHU5qaqK9+98hkkfjAJg99tO59kz/s7McZN59bK72Pn6E9jszAOY8O4nvH/XM/V4lAKfc7Ikke9uQQstFDEE2Ax4JaXUNyLWAS5LKe27lFVp3aK33eq0mIqy8voOQSVo1/br1XcIKjH/OD9vD2I1YrddNH7pC6lROnrUv1aIB/uc3uun9Xp+/PtP7izZ31Ntn3MyK6U0KyKIiGYppfcjYu2CRiZJkiQ1QKV+O9/6VNvGyaiIaAs8CDwVEZOATwsXliRJkqTGplaNk5TSj7PJiyJiINAG8DHEkiRJkurMUhsnEVEODE0prQOQUnq24FFJkiRJDZSduvJb6q2EU0rzgA8iYpWlLStJkiRJy6u2Y07aAUMj4jVgxvzClNJeBYlKkiRJaqC8lXB+tW2cnF/QKCRJkiQ1erVtnOyRUjqrekFE/A5w/IkkSZKkOrHUMSeZH9RQtntdBiJJkiQ1Bqme/5WyJWZOIuJY4Dhg9Yh4p9pbrYCXChmYJEmSpMZlad267gAeBy4Hfl2tfFpK6auCRSVJkiQ1UA6Iz2+JjZOU0hRgSkSctchbLSOiZUrps8KFJkmSJKkxqe2A+EfJPS8mgEpgNeADYL0CxSVJkiSpkalV4ySltEH1+YjYmNxYFEmSJEnLoKrEB6XXp9rerWshKaVBwPfrOBZJkiRJjVitMicRcWq12TJgY2B0QSKSJEmSGjDzJvnVdsxJq2rTc8mNQbmv7sORJEmS1FjVdszJxQAR0TylNLOwIUmSJElqjGo15iQitoyIYcD72fyGEXF9QSOTJEmSGqAqUr2+SlltB8RfC+wKTARIKb0NbFeooCRJkiQ1PrW+W1dK6fNFiubVcSySJEmSGrHaDoj/PCK2AlJENAFOBt4rXFiSJElSw1RV3wGUsNpmTo4Bjgd6AF8AfbN5SZIkSaoTtb1b1wTg4ALHIkmSJDV4qcQHpdenJTZOIuKCJbydUkqX1HE8kiRJkhqppWVOZtRQ1gI4AugA2DiRJEmSVCeW2DhJKV09fzoiWpEbCP9z4C7g6nzrSZIkSaqZA+LzW+qYk4hoD5xKbszJrcDGKaVJhQ5MkiRJUuOytDEnVwH7Av2BDVJK04sSlSRJktRAOSA+v6XdSvg0oDtwHjA6IqZmr2kRMbXw4UmSJElqLJY25qTWT5CXJEmSpO+itk+IlyRJklQHHBCfn5kRSZIkSSXBzIkkSZJURFXJAfH5mDmRJEmSVBJsnEiSJEkqCXbrkiRJkorITl35mTmRJEmSVBLMnEiSJElFVGXuJC8zJ5IkSZJKgo0TSZIkSSXBbl2SJElSESW7deVl5kSSJElSSbBxIkmSJKkk2K1LkiRJKqKq+g6ghJk5kSRJklQSzJxIkiRJReRzTvIzcyJJkiSpJNg4kSRJklQS7NYlSZIkFZHPOcnPzIkkSZKkkmDmRJIkSSoibyWcn5kTSZIkSSXBxokkSZKkkmC3LkmSJKmIUnJAfD5mTiRJkiSVBDMnkiRJUhH5hPj8zJxIkiRJKgk2TiRJkiSVBLt1SZIkSUXkc07yK3jjpGvz9oXehVZAJ1WuU98hqAStO2t2fYegEnP5b8fVdwgqQefcvUd9hyCpQMycSJIkSUWUHBCfl2NOJEmSJJUEGyeSJEmSSoLduiRJkqQi8jkn+Zk5kSRJkrRARKwcEQMjYlhEDI2Ik7Py9hHxVEQMz362y8ojIv4UESMi4p2I2Hh5923jRJIkSSqilFK9vmphLnBaSmldYAvg+IhYF/g1MCCltCYwIJsH2B1YM3sdBfx1eX83Nk4kSZIkLZBSGpNSGpRNTwPeA3oAewO3ZovdCuyTTe8N3JZyXgHaRkS35dm3jRNJkiSpEYmIoyLijWqvo5awbC9gI+BVoEtKaUz21ligSzbdA/i82mqjsrJl5oB4SZIkqYjq+wnxKaX+QP+lLRcRLYH7gF+llKZGRPVtpIio85H9Zk4kSZIkLSQimpBrmNyeUro/Kx43v7tW9nN8Vv4FsHK11XtmZcvMxokkSZKkBSKXIrkJeC+l9Idqbz0EHJZNHwb8p1r5odldu7YAplTr/rVM7NYlSZIkFVEq/eecbA38DBgSEYOzsnOAK4B7IuII4FPgwOy9x4A9gBHATODny7tjGyeSJEmSFkgpvQBEnrd3qmH5BBxfF/u2cSJJkiQVkU+Iz88xJ5IkSZJKgo0TSZIkSSXBbl2SJElSEeWGaKgmZk4kSZIklQQzJ5IkSVIROSA+PzMnkiRJkkqCjRNJkiRJJcFuXZIkSVIRrQBPiK83Zk4kSZIklQQzJ5IkSVIRVXkr4bzMnEiSJEkqCTZOJEmSJJUEu3VJkiRJRWSnrvzMnEiSJEkqCWZOJEmSpCLyCfH5mTmRJEmSVBJsnEiSJEkqCXbrkiRJkorIbl35mTmRJEmSVBLMnEiSJElFlHxCfF5mTiRJkiSVBBsnkiRJkkqC3bokSZKkInJAfH5mTiRJkiSVBBsnkiRJkkqC3bokSZKkIkp268rLzIkkSZKkkmDmRJIkSSoin3OSn5kTSZIkSSXBxokkSZKkkmC3LkmSJKmIfM5JfmZOJEmSJJUEMyeSJElSETkgPj8zJ5IkSZJKgo0TSZIkSSXBbl2SJElSETkgPj8zJ5IkSZJKgpkTSZIkqYiSmZO8zJxIkiRJKgk2TiRJkiSVBLt1SZIkSUVU5XNO8jJzIkmSJKkkmDmRJEmSisgB8fmZOZEkSZJUEmycSJIkSSoJduuSJEmSisgB8fnZOFlOl/3xAnb4wTZMnDCJH253EAC77bUTJ55xFKuvtRr773IY77793oLljz75cPY/eG/mzavit+dcxQsDX1lsmz1X6c41/S+jbfs2DH37Pc447gLmzJlbtGPSd1ferAn7/vs8yptWEOXlfPTYa7z2h/vpufV6bHXuT4myYM6MWQw4rT9TPhm3YL3Vd9+M3fufzD17ns/4d0Yutt1V+vVh24t+RpSXMezOZxh0/cPFPCzVhbJgs/9ewTdjv+KdQ35Hu23XZ40LDoGyMubNmMV7J/2Frz8ZR9stvsealxxGi3VXZejR1/LlI6/WuLlWfVbje386nrLKpkwc8BbDz725yAek76KiWRN+cff5VDSroKy8nKGPv8bAa+5j79/9kh59VgOCiSPH8sDpNzB75jesuvk67H7BIXRZZxXuPfE6hj3+Wo3b7bZ+L/b9/TFUVDZh+MC3eezi24p7YPrObn/qVe57bhApwX7bbcQhu2zBf18fxl//8ywjx3zJ7ecdyXqrdQfg0ZeHcOsTLy1Y98NR47jrwqNYZ5WuC21zyvSvOfOGfzN6whS6d2zDVcfuT+sWKxX1uKTaslvXcrr/roc54icnLlQ2/L2POOHwM3n95bcWKl99rdXYc59d2GObAznyoBO56He/pqxs8V/96RecyC033MEPNv8xUyZPY/+D9y7oMajuzftmDg8edBl37Xoud+92Lqv060OXjVan32WH89RJ13P3bufy4X9eZtOTvv3bNmlRSZ8jdmXsoBE1bjPKgu1/exgPH3old+x4JmvtvQXt1uxerENSHVn5l3swY/gXC+bX/t2RDD3uz7y+05mMu/8Fep2yHwCzvpjAsJOvZ9z9Lyxxe2tf+UveP+1GXtniJJqv1pX2O/YtaPyqW3O/mcMt/3cp1+9+DtfvcQ5rbt+HnhutwROX/CtXtvvZTBk9ge8ftgsAU0ZP4IHTb2TIf15a4nZ/9Ntf8J+z/84f+51Gh9W6sma/DYtxOKojw0eN577nBnH7eUdy78VH89zbw/ls3Fes0aMT1xx/AJustepCy++55Qbcc/HR3HPx0Vz6y33o0bHdYg0TgH889gKbf281Hr7iBDb/3mrc9NiLxTok5ZHq+V8ps3GynN54+S2mTJq6UNlHwz9h5EefLrbszrtvz6MP/pc5s+cw6rPRfPrJ5/TZeL3Flttym8144uEBADxw9yPsvEe/gsSuwpoz8xsAyirKKauogAQpQdOWuatUzVqtxIxxkxcs//3T92fQ9Y8w75s5NW6vS9/VmfLJOKZ+9iVVc+Yx/KFX6L3LJoU/ENWZZt3a0+EHGzPm9gELylKCila5OlHRujnfjJsEwKzPv2TGsM+gKv+XR9PObSlvuRJT3xwOwNh7n6PT7psV8AhUCLOzz4ryinLKKsohJb6Z/vWC9ysqm5Kyrh+TR01g3PufL5ivSctObWnWaiVGvZW70DH4/udZx8+KFcrIMRPYYLUerNSsCRXlZWyy9qoMGPQevbt3ole3jktc9/FX32W3zRc/twAY+NaH7LV1rqG619YbMnDQB3Ueu1RX7NZVBF26dWbwG0MWzI8dPZ4u3TovtEy79m2YOnUa8+bN+3aZrgsvoxVDlAUHPvZb2vTqwpBbn2Lc4I94+sy/86PbTmfurDnMnvY19+59EQCd1u9Fq+7t+fTpwWx8zJ41bq9F13ZMG/3VgvnpY76iy0arF+NQVEfWvORwPvrNvyhv+W03ivdPvYENbz+bebNmM2/a17yxx7m13l6zbu35ZszEBfOzRk+kWbf2dRmyiiDKgmMeuZT2q3bhtX8+xajBHwGwz1VHsVa/vnw54gue/O3ttd5e667tmDrm28+KqWO+onUX68WKZI0enfjz/U8zefpMmjVpwgtDhrNur9plyp98bRjXnnhQje99NXU6ndq2AqBjm5Z8NXV6ncUs1bVaZU4iYq2IGBAR72bzfSLivMKGJq2YUlXi7t3O5ZbNT6JL39Vpv3ZP+h65Gw8f+ntu2fwk3rvnOba54GCIYJsLDuaFS+6o75BVQB1+sDGzJ0xh2iJjiVY+ek/ePvhyXtroWMbcNZA1f3NoPUWo+pKqEn/d4xyu3vJEem64Op3X6gnAg2f056rvH8+XI75g/R9tUc9Rqph6d+/Ez3ffmmOuvp3jrrmdtVfuSnks/VTtnY9GUdm0CWv2XPpFzYiAiLoIV99BVUr1+iplte3W9TfgbGAOQErpHeAn+RaOiKMi4o2IeGPKrC+/e5QruHFjxtOtR5cF8127d2bcmPELLTPpqym0bt2K8vLyb5cZu/AyWrHMnjqTL14axqr9NqTjuqswLrsqOvzhV+i2yZo0bVlJ+7V78uN7zuXQl66hy0ars+c/TqVzn9UW2s6MsZNo1f3bq58tu7VnxthJRT0WLb82m69Nx103ZcvXr2O9G39Fu63Xp8+/fk2r9VZlajbOaNx/XqLNpmvXepvfjPmKZt06LJiv7N6Bb6pdMdeKZdbUmYx8eRhrbt9nQVmqSgx5+BXW3W3zWm9n6thJtK6WQWvdrT1Tx1kvVjT7brcRd134S27+9eG0blHJql2Xnv168rWh7P79mrt0AbRv3ZIvJ08D4MvJ02jfqkWdxSvVtdo2TpqnlBa9NUje20illPqnlDZNKW3aprLT8kfXQAx44jn23GcXmjRtQs9VutNrtZV5Z9DQxZZ75cU32O1HOwHw44N+yIDHny12qPqOKtu3omnr5gCUVzZh5e02YNKIL2jaqjltV8sNUlx52/WZNOILZk/7mps2PJbbtjqF27Y6hXFvfcSjv/jDYnfrGvf2x7Tp1ZVWK3eirEk5a+61BSOfGlT0Y9Py+fjSO3lpo2N5ebMTGHr0tUx68V2GHHYl5a2as1LvbgC0377PQoPll2b2+MnMm/41rTdZE4CuB2zHhCfeKEj8Kozm7VtRmX1WVDRrwurbrM+Ej8fQftVvL2Sts/PGTPhodK23Of3LyXwz7Wt6brQGAH333Zb3//tm3Qaugps4dQYAYyZOYcCb77P7FhsscfmqqsSTrw9jt83Xz7tMv43W4qEX3wbgoRffZoeN1qq7gKU6VtsxJxMiYnXIDe+PiP2BMQWLagXwhxsvZfOtN6Fd+7Y89/aj/OnK/kyZNIXzLz+D9h3a0f+Oa3lv6IccceCJjPjgYx576H88/sK9zJ03j4t/fSVVVVUA/O3OP3Lury5h/LgJ/P43f+aa/pfxq3OOZdiQD7j39v/U81FqWbXo3JadrzmaKC8jyoIRD7/KJwMGM/Csm9i9/8mkqiq+mTKTAaf3X/J2urRlhyuP5JHDfk+aV8Vz59/K3v86M3cr4buf5asPa38iq9KT5lXx/mk3ssE/TiNVVTF38gzeO+WvALTquzob3Hw6Tdq2oOMum7DaGQfy2vanAbDZgCt5faczAfjgrL/zvT8dR3llUyYOGMzEAW/l3Z9KT6vObdn36mOIstxnxdBHX+XDpwdzxL0X0KzlShAw9r3PeOS83C2iu/fpzU9vPIWV2jRn7Z02YsdT9uO6Xc4C4NjHLuOve5wDwCPn38yPf380TSqbMvyZtxn+zNv1doxaPqf95R6mTP+aivJyzjlkd1o3r2TAm+9zxR2PM2naTE74452svXIXbjjtEADe/PBTurZvTc/O7RbazkU3P8wB/TZhvdW684s9tuaMv/6bB58fTLcOuVsJq36V+h2z6lMs6c4fCxaK6A30B7YCJgEjgYNTSovfmmoRa3Xa1N++FnNS5Tr1HYJK0LqzZ9d3CCoxz6zUpL5DUAk65/Y96jsElajKrQ9eIQbUrNlpk3o9Px7+5Zsl+3uqbebk05TSzhHRAihLKU0xD/dZAAAOcElEQVQrZFCSJElSQ1Xqg9LrU23HnIyMiP7AFoD3n5MkSZJU52rbOFkH+B9wPLmGynURsU3hwpIkSZLU2NSqcZJSmplSuieltC+wEdAa8FZSkiRJ0jJK9fyvlNU2c0JEbB8R1wNvApXAgQWLSpIkSVKjU6sB8RHxCfAWcA9wRkppRiGDkiRJkhqqlKrqO4SSVdu7dfVJKU0taCSSJEmSGrUlNk4i4syU0pXApRGxWAe1lNJJBYtMkiRJUqOytMzJe9nPNwodiCRJktQYVJX4oPT6tMTGSUrp4WxyZkrp3urvRcQBBYtKkiRJUqNT27t1nV3LMkmSJElLkFKq11cpW9qYk92BPYAeEfGnam+1BuYWMjBJkiRJjcvSxpyMJjfeZC9yzzeZbxpwSqGCkiRJktT4LG3MydvA2xFxe0rJTIkkSZL0HTkgPr+ldeu6J6V0IPDWIrcSDiCllPoUNDpJkiRJjcbSunWdnP38YaEDkSRJkhqDUh+UXp+WeLeulNKYbHIC8HlK6VOgGbAhufEokiRJklQnansr4eeAyojoAfwX+BlwS6GCkiRJktT4LK1b13yRUpoZEUcA16eUroyIwYUMTJIkSWqIquzWlVdtMycREVsCBwOPZmXlhQlJkiRJUmNU28bJr8g9Ef6BlNLQiOgNDCxcWJIkSZIam1p160opPQs8GxEtI6JlSulj4KTChiZJkiQ1PMnnnORVq8xJRGwQEW8BQ4FhEfFmRKxX2NAkSZIkNSa1HRB/I3BqSmkgQET0A/4GbFWguCRJkqQGyeec5FfbMSct5jdMAFJKzwAtChKRJEmSpEaptpmTjyPifOCf2fwhwMeFCUmSJElSY1TbxskvgIuB+4EEPJ+VSZIkSVoGVQ6Iz2uJjZOIqASOAdYAhgCnpZTmFCMwSZIkSY3L0jIntwJzyGVKdge+R+6ZJ5IkSZKWgwPi81ta42TdlNIGABFxE/Ba4UOSJEmS1Bgt7W5dC7pwpZTmFjgWSZIkSY3Y0jInG0bE1Gw6gJWy+QBSSql1QaOTJEmSGpgqu3XltcTGSUqpvFiBSJIkSWrcansrYUmSJEl1wAHx+dX2CfGSJEmSVFA2TiRJkiSVBLt1SZIkSUXkE+LzM3MiSZIkqSSYOZEkSZKKyAHx+Zk5kSRJklQSbJxIkiRJKgl265IkSZKKyCfE52fmRJIkSVJJMHMiSZIkFVHyVsJ5mTmRJEmSVBJsnEiSJEkqCXbrkiRJkorIAfH5mTmRJEmSVBJsnEiSJEkqCXbrkiRJkooo2a0rLzMnkiRJkkqCmRNJkiSpiHzOSX5mTiRJkiSVBBsnkiRJkkqC3bokSZKkInJAfH5mTiRJkiSVBDMnkiRJUhGZOcnPzIkkSZKkkmDjRJIkSVJJsFuXJEmSVER26srPzIkkSZKkkhAOyCmeiDgqpdS/vuNQabFeqCbWC9XEeqGaWC/UkJg5Ka6j6jsAlSTrhWpivVBNrBeqifVCDYaNE0mSJEklwcaJJEmSpJJg46S47A+qmlgvVBPrhWpivVBNrBdqMBwQL0mSJKkkmDmRJEmSVBJsnEiSJEkqCStU4yQi5kXE4GqvXy9l+XOKFVu1ffaLiEeWYfnDI6IqIvpUK3s3InrVcVx9I2KPavN7Le33tyJqwHXky+x4hkXEL5djn8dExKHVtte92nt/j4h1l3Wb+lZEpIi4utr86RFxUQH2c84i8y/V9T5UGHVZRyKibUQct5zrfhIRHZdnXdW9at9Z70bEvRHRfBnX7x4R/86mG8X3vBq+FapxAnydUupb7XXFUpav8cQzckrp2EcB5xZ4H32BBR9aKaWHavH7WxE11Dpyd0qpL9APuCwiuizLyimlG1JKt2WzhwPdq713ZEppWF0F2kh9A+xbhJO+heprSmmrAu9Pdacu60hboMbGSURU1MH2VTzzv7PWB2YDxyzLyiml0Sml/bPZxvI9rwaulE6+lktEtImIDyJi7Wz+zoj4ZURcAayUXZG4PSJ6ZcvdBrwLrBwRZ0TE6xHxTkRcnK3fKyLej4hbIuLDbN2dI+LFiBgeEZtny7WIiH9ExGsR8VZE7L1IXGXZ8p2qzY+YP7+IR4D15h/DItvZJSJejohB2VWVlln5Hlmcb0bEn+ZfiY+IzbPl34qIlyJi7YhoCvwGOCj7fRyUXT2/Lvv9fTr/RDw7rs8joklErB4RT2T7eD4i1qmTP1qRNZA6AkBKaTzwEbBqROyUbXdItp9m2XauiFyG5Z2I+H1WdlHkrtTuD2wK3J4d90oR8UxEbBq57MpV1eI7PCKuy6YPyY5jcETcGBHldfTnaSjmkrtbzimLvhERnSLivqwevR4RW1crfyoihkYue/VpZCeuEfFg9v9uaEQclZUtVF+zsunZz7siYs9q+7wlIvaPiPKIuKpaHT664L8J5bM8deSiiDi92nLzs+pXAKtndeGqyGVjn4+Ih4Bh2bKL1SGVvOeBNSKiffb3eyciXomsZ0VEbB/f9gp4KyJaZd9H70Yj/55XA5NSWmFewDxgcLXXQVn5D4CXgZ8AT1Rbfnq16V5AFbBFNr8LuS+KINdIewTYLltuLrBBVv4m8I9sub2BB7P1LwMOyabbAh8CLchd2X4kK78Q+FW1/d1XwzEdDlwHHArcmpW9m8XREXgOaJGVnwVcAFQCnwOrZeV3Vttna6Aim955/j7n72fR/WbT/wF2yKYPAv6eTQ8A1symvw88Xd91oDHXkWy6NzCeXObjc2CtrPw24FdAB+ADvr0TX9vs50XA6dn0M8Cm1bb/DLkGSydgRLXyx4FtgO8BDwNNsvLrgUPr+29dSi9gevZ/7xOgDXA6cFH23h3ANtn0KsB72fR1wNnZ9G5AAjpm8+2znyuR+zzosGh9rT4P/JhvPz+aZnVjJXJPjT4vK28GvEH2ueFrhagjC/7fZvPzvxt6Ae9WK+8HzKj+t11CHfpkfj3zVf+vav+HK8h9Fx8L/Bm4MCvfERicTT8MbJ1Nt8zWWVAXaCTf874a/mtFS/9+nXJdWxaSUnoqIg4A/gJsuIT1P00pvZJN75K93srmWwJrAp8BI1NKQwAiYigwIKWUImIIuQ+C+evvVe2qViW5L5Xq/kHuA+Fa4BfAzUuI7Q7g3IhYrVrZFsC6wIsRAbmTjpeBdYCPU0ojs+XuJHcSArkvvVsjYk1yJztNlrDP+e4m92E1kNzJ+/WRy9BsBdyb7RtyJzelrqHWkYMiYhtyXUOOJteQGJlS+jB7/1bgeHInvLOAmyKXTav12JaU0pcR8XFEbAEMJ1fPXsy2uwnwelYXViLXQFI1KaWpkcu6nQR8Xe2tnYF1q/0/ap39/9qGXKOClNITETGp2jonRcSPs+mVydW7iUvY/ePAH7Ps2W7AcymlryNiF6BPljGD3OfDmsDIPNtRAS1HHVkWr1X7ToBlr0OqHytFxOBs+nngJuBVYD+AlNLTEdEhIlqT+zz+Q5Y5vT+lNKpanVmahvQ9rwZuRWuc1ChLVX4PmAm0IzeGoyYzqq8GXJ5SunGRbfUidwI4X1W1+Sq+/Z0FsF9K6YNF1l8wFiCl9HlEjIuIHYHNgYPzHUNKaW7kBkuetUiMT6WUfrrIPhY7+a7mEmBgSunH2bE8s4Rl53uI3DiG9uROQp8md4V/ck0n+iuiBlBH7k4pnVBtGzU2sLJ6tDmwE7A/cAK5K2+1dRdwIPA+8EDW4ApyV+XPXobtNFbXAoNYuJFZRi4bN6v6gvlOKiKiH7mT1S1TSjMj4hlyDdu8UkqzsuV2JXcCctf8zQEnppSeXNYDUcEsSx2Zy8Ldr5dUDxZ8di1PHVK9WeyCWr7PhpTSFRHxKLlxJS9GxK7kLkbVRoP/nlfDscKPOcmcArwH/B9wc0TMzxbMqTa9qCeBX8S3Yzh6RETnZdjnk8CJ2YkbEbFRnuX+DvwLuDelNG8p27yF3BfK/DEHrwBbR8Qa2T5aRMRa5Lrt9I5v7+h1ULVttAG+yKYPr1Y+DWhV005TStOB14E/kutuNC+lNBUYmWUb5g8QX1LGodQ1lDoy3wdAr/l1A/gZ8GwWa5uU0mPkjrmmv1neugA8QK5r2k/59gR3ALD//GPP+kOvWss4G5WU0lfAPcAR1Yr/C5w4f6baxYUXyTUEyTIc7bLyNsCk7KRyHXIZ1PmWVF/vBn4ObAs8kZU9CRw7f52IWCsiWizn4akOLGMd+QTYOCvbGJifWV/S/2FYch1S6Xue7EJV1tCckGXdVk8pDUkp/Y7cd/ai40Ma+/e8GogVrXEyfzDo/NcVkRvkfCRwWkrpeXJjNM7Llu8PvJOlQBeSUvovua5UL2ddcf7Nkj/sF3UJuS5T72Tdei7Js9xD5LoDLalL1/yYZgN/Ajpn81+Sa2DcGRHvkHXpSil9Te5OLU9ExJvkPpCmZJu5Erg8It5i4czYQHLdBgZHRPXGzHx3A4dkP+c7GDgiIt4GhpI7aS11DbqOVIttFrkT0Xuz2KqAG7L4HsnqywvAqTWsfgtwQ/b7WWmR7U4i14hbNaX0WlY2jNzv67/Zdp8CutU21kboanLjxeY7Cdg0coNbh/Ht3XguBnaJiHeBA4Cx5P4vPwFURMR75AY+v1JtW3nrK7kT3O2B/2WfJZBr+A4DBmX7uZEGkjFfwdW2jtwHtM8+P04gN26NlNJEclfO341qN7GoZkl1SKXvImCT7PP2CuCwrPxX2d/8HWAOue6c1TWW73k1cPMHzapAImJT4JqU0rZ1vN2WKaXp2VX5vwDDU0rX1OU+VByFqiMqbZEbHzIv64q3JfBXu1dIkho7r6AVUOQefnQsSxhr8h38MiIOIzdI/i1yV0S1gilwHVFpWwW4JxsPNRtY5odrSpLU0Jg5kSRJklQSVrQxJ5IkSZIaKBsnkiRJkkqCjRNJkiRJJcHGiSRJkqSSYONEkiRJUkn4f4S7tP0v7W3vAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 1080x864 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    }
  ]
}