Fine_Tuning_BERT_on_PAWS_X_using_Pytorch.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Copy of Fine-Tuning BERT on PAWS-X using Pytorch.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "e57c3d7c9e1d4b1f8492f59fd3bd35a8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f67e7b66e057408d8c7c00926c43771b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_746cd7b92d2a44d2a474d60f4f58819a",
              "IPY_MODEL_f88c3094aae44297aed0a9bce94214e9"
            ]
          }
        },
        "f67e7b66e057408d8c7c00926c43771b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "746cd7b92d2a44d2a474d60f4f58819a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_39219d65d33d460fb8569830e3c580e7",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e70de0859658432faacaafec55fe597b"
          }
        },
        "f88c3094aae44297aed0a9bce94214e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_449f31c913d24ba4ae8fb313d9b0a567",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 241kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_30452fc30a05406f9fb2f41161a2fe08"
          }
        },
        "39219d65d33d460fb8569830e3c580e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e70de0859658432faacaafec55fe597b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "449f31c913d24ba4ae8fb313d9b0a567": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "30452fc30a05406f9fb2f41161a2fe08": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "962efbff33764c14a5f2ff46c485a3c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5c370023cad84f709ce595b5deeaed96",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4d3163ba1d244ddb8d7bc6513017591a",
              "IPY_MODEL_ac755143edac498698c949dcee6b9624"
            ]
          }
        },
        "5c370023cad84f709ce595b5deeaed96": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4d3163ba1d244ddb8d7bc6513017591a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d34a439b719f4961bfd37148840ffd93",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 28,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 28,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_eae8927c9114464ab368f94fb9bc3b9d"
          }
        },
        "ac755143edac498698c949dcee6b9624": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ad276274f11f4f8eb05cca79a6c019c9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 28.0/28.0 [00:00&lt;00:00, 65.9B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f00a8f0023bf47dea2d3fcaa2c925ca9"
          }
        },
        "d34a439b719f4961bfd37148840ffd93": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "eae8927c9114464ab368f94fb9bc3b9d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ad276274f11f4f8eb05cca79a6c019c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f00a8f0023bf47dea2d3fcaa2c925ca9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "737c993d116b4923b87297a7118edf94": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_26a68c675e234354b6029728f4f867fc",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_23aa8345f1ce4b808d1cdb30f559bff2",
              "IPY_MODEL_1bda39694d3045819ac4ff0da845307b"
            ]
          }
        },
        "26a68c675e234354b6029728f4f867fc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "23aa8345f1ce4b808d1cdb30f559bff2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_913258ab441d44b6a56222bc067bf072",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 466062,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 466062,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_231ee0da726843a3a69fe701c706ec40"
          }
        },
        "1bda39694d3045819ac4ff0da845307b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_625cc535dfb147d0ac50cd69cfb42279",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 466k/466k [00:00&lt;00:00, 2.66MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6ba9bd21d5e14461be1d8d1ad74d5f48"
          }
        },
        "913258ab441d44b6a56222bc067bf072": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "231ee0da726843a3a69fe701c706ec40": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "625cc535dfb147d0ac50cd69cfb42279": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6ba9bd21d5e14461be1d8d1ad74d5f48": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1fbb9d0e5d9c48c58680ccea3718110d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f6311cde5f0d4db5841a8d1b1871e08d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_836d6676fdc244a7b9b99d201dd71280",
              "IPY_MODEL_8d1814f92f944d67928c95c7525e26f0"
            ]
          }
        },
        "f6311cde5f0d4db5841a8d1b1871e08d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "836d6676fdc244a7b9b99d201dd71280": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6ac31ece46cd4b22963a96d213d2796d",
            "_dom_classes": [],
            "description": "Downloading: ",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2529,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2529,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9f644277b15f4fa5a3ae8285325e3b6b"
          }
        },
        "8d1814f92f944d67928c95c7525e26f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_451cfd2d8fcf4bbba82180bddf5afad1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 6.57k/? [00:02&lt;00:00, 2.39kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c3d638e64b274f9f827dad41517b3e5f"
          }
        },
        "6ac31ece46cd4b22963a96d213d2796d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9f644277b15f4fa5a3ae8285325e3b6b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "451cfd2d8fcf4bbba82180bddf5afad1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c3d638e64b274f9f827dad41517b3e5f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "089845e1098c42fb90a6da033adc1e7b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6ad42df6c502431e8d61b5d6129ba8a8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_da7bae5587574cdbbfc69f18aa9e61ec",
              "IPY_MODEL_559bbfb44f6144aa8a2e24ec16e7788d"
            ]
          }
        },
        "6ad42df6c502431e8d61b5d6129ba8a8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "da7bae5587574cdbbfc69f18aa9e61ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f0f4e021f3c647a98ad23c1e0a5cd60c",
            "_dom_classes": [],
            "description": "Downloading: ",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1648,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1648,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8ee56979b5ae4f069a6800915a47a021"
          }
        },
        "559bbfb44f6144aa8a2e24ec16e7788d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_80e99ded59994841939c942a79306938",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 17.6k/? [00:00&lt;00:00, 304kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dec6fb32747540d68d0514cccd1026c3"
          }
        },
        "f0f4e021f3c647a98ad23c1e0a5cd60c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8ee56979b5ae4f069a6800915a47a021": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "80e99ded59994841939c942a79306938": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dec6fb32747540d68d0514cccd1026c3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3d5732ec83a74cf292eaac134d719897": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_64df45611cd5498eab67e8fd5cbc07d3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f0601b13757c4f0e93bd100ae2de4009",
              "IPY_MODEL_5928c64e567a4cfc8795c28bf11de3d7"
            ]
          }
        },
        "64df45611cd5498eab67e8fd5cbc07d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f0601b13757c4f0e93bd100ae2de4009": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e75f82437301484f91d168cf406c0610",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 30282057,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 30282057,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0853351f3e8f4cf8a9113815478c92fb"
          }
        },
        "5928c64e567a4cfc8795c28bf11de3d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_faddabf3315a4c42b46c77aa066667c6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 30.3M/30.3M [00:02&lt;00:00, 14.2MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_52879479ca034034850d277c6f4fc216"
          }
        },
        "e75f82437301484f91d168cf406c0610": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0853351f3e8f4cf8a9113815478c92fb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "faddabf3315a4c42b46c77aa066667c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "52879479ca034034850d277c6f4fc216": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "094c8373e1e9462fb1334c0c70bc0e48": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_36b513532d0f42ff807c985dd2d159ac",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_71bf6fa3c5844adbafcfe7b8176417f7",
              "IPY_MODEL_de9dd6dae5d842b288ac668f184533ee"
            ]
          }
        },
        "36b513532d0f42ff807c985dd2d159ac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "71bf6fa3c5844adbafcfe7b8176417f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7961f9dd530b4de8930d85f6d9ad49e9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f0516e85be9c4f5f88cad5c892cb8530"
          }
        },
        "de9dd6dae5d842b288ac668f184533ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_713cd2ba9824442cb84987c6700e008f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 49401/0 [00:01&lt;00:00, 31857.74 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4164df2707fb44eb8a88963a35011474"
          }
        },
        "7961f9dd530b4de8930d85f6d9ad49e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f0516e85be9c4f5f88cad5c892cb8530": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "713cd2ba9824442cb84987c6700e008f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4164df2707fb44eb8a88963a35011474": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "234d7de0abf6407bbd2d115afe557b7a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8cce77cc40a64c3eb1ed9d6c0a6b6978",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c0ddc430cbfd4adcb6202a415225fec1",
              "IPY_MODEL_1d191356dcad4df29fd8a28974ce2cdb"
            ]
          }
        },
        "8cce77cc40a64c3eb1ed9d6c0a6b6978": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c0ddc430cbfd4adcb6202a415225fec1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1497d7e17c214ebdb26911ffdef97aee",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2287732f01ea4e2a8eab379780cce890"
          }
        },
        "1d191356dcad4df29fd8a28974ce2cdb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_95098afa765d433d8ad4ebfc33c137a5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2000/0 [00:00&lt;00:00, 19338.77 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b77b869cdc304f05a1765bb274131255"
          }
        },
        "1497d7e17c214ebdb26911ffdef97aee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2287732f01ea4e2a8eab379780cce890": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "95098afa765d433d8ad4ebfc33c137a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b77b869cdc304f05a1765bb274131255": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "298d0ef1ab41422289c5deced2506c58": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_cec84b95365d4518b076d5fc03fe23a8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8c55a6709ab449288188f50b647fc129",
              "IPY_MODEL_1e2ca6fe8ab24309b5619a2f642efbcf"
            ]
          }
        },
        "cec84b95365d4518b076d5fc03fe23a8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8c55a6709ab449288188f50b647fc129": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d0470c10a73747849d4a6906665c504f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9a9b46dbadd1442eac971250a788df30"
          }
        },
        "1e2ca6fe8ab24309b5619a2f642efbcf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_74e84d085870496baf20afb221d93401",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2000/0 [00:00&lt;00:00, 10848.43 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_26d7c7e05aa44d9d9d9e0b676d8fe245"
          }
        },
        "d0470c10a73747849d4a6906665c504f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9a9b46dbadd1442eac971250a788df30": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "74e84d085870496baf20afb221d93401": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "26d7c7e05aa44d9d9d9e0b676d8fe245": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "af3531e3c8124027bfb237c86d5c094f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_50070c08115d4103ab27aea687535f23",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2646ad7872834cd083a1b6c092942e01",
              "IPY_MODEL_0f548e00e6f94e1f968d845e2c18b084"
            ]
          }
        },
        "50070c08115d4103ab27aea687535f23": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2646ad7872834cd083a1b6c092942e01": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9b00175550304222a33e9ea2e9f0d7fa",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 625,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 625,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7ffe02e27f8a4e7c832604e97b173c4c"
          }
        },
        "0f548e00e6f94e1f968d845e2c18b084": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_27abed83dd53475f850f1e56d9168cf1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 625/625 [00:29&lt;00:00, 21.5B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f9ff3c83c055436d8e1be8e8f9b4bfc1"
          }
        },
        "9b00175550304222a33e9ea2e9f0d7fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7ffe02e27f8a4e7c832604e97b173c4c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "27abed83dd53475f850f1e56d9168cf1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f9ff3c83c055436d8e1be8e8f9b4bfc1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3c16ccfb58e040618a6ef57eb7e820e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_702ab50ea5fd4f058cd0d7b414d12098",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_321be65546654aa9ae7943bb89ef5713",
              "IPY_MODEL_b4f09bfb735a4091a15532252e85f71d"
            ]
          }
        },
        "702ab50ea5fd4f058cd0d7b414d12098": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "321be65546654aa9ae7943bb89ef5713": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3fad0e0f98454367a6bab2949c7d4837",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 672271273,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 672271273,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_743b4b8b64ce41d898e62be6374ce5b7"
          }
        },
        "b4f09bfb735a4091a15532252e85f71d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a76fa1f419104733881d53fe380bae02",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 672M/672M [00:12&lt;00:00, 52.5MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b157dea0e639449e964f73b294b01720"
          }
        },
        "3fad0e0f98454367a6bab2949c7d4837": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "743b4b8b64ce41d898e62be6374ce5b7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a76fa1f419104733881d53fe380bae02": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b157dea0e639449e964f73b294b01720": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HT0bVJ4xtwzk"
      },
      "source": [
        "This Notebook is based on: https://colab.research.google.com/drive/1pTuQhug6Dhl9XalKB0zUGf4FIdYFlpcX"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RX_ZDhicpHkV"
      },
      "source": [
        "# 1. Setup"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nSU7yERLP_66"
      },
      "source": [
        "## 1.1. Using Colab GPU for Training\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DEfSbAA4QHas",
        "outputId": "2c538308-951b-42fd-fe9e-ddb634e03b07"
      },
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "# Get the GPU device name.\n",
        "device_name = tf.test.gpu_device_name()\n",
        "\n",
        "# The device name should look like the following:\n",
        "if device_name == '/device:GPU:0':\n",
        "    print('Found GPU at: {}'.format(device_name))\n",
        "else:\n",
        "    raise SystemError('GPU device not found')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found GPU at: /device:GPU:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cqG7FzRVFEIv"
      },
      "source": [
        "In order for torch to use the GPU, we need to identify and specify the GPU as the device. Later, in our training loop, we will load data onto the device. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oYsV4H8fCpZ-",
        "outputId": "c6f36978-4ae2-4eb0-e5c1-fa82d3dc9ac8"
      },
      "source": [
        "import torch\n",
        "\n",
        "# If there's a GPU available...\n",
        "if torch.cuda.is_available():    \n",
        "\n",
        "    # Tell PyTorch to use the GPU.    \n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "\n",
        "# If not...\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla P4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2ElsnSNUridI"
      },
      "source": [
        "## 1.2. Installing the Hugging Face Library\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0NmMdkZO8R6q",
        "outputId": "faf39588-10c5-4bb7-8c49-f2698e93a9e8"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ed/d5/f4157a376b8a79489a76ce6cfe147f4f3be1e029b7144fa7b8432e8acb26/transformers-4.4.2-py3-none-any.whl (2.0MB)\n",
            "\u001b[K     |████████████████████████████████| 2.0MB 5.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 21.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (3.7.2)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/71/23/2ddc317b2121117bf34dd00f5b0de194158f2a44ee2bf5e47c7166878a97/tokenizers-0.10.1-cp37-cp37m-manylinux2010_x86_64.whl (3.2MB)\n",
            "\u001b[K     |████████████████████████████████| 3.2MB 33.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp37-none-any.whl size=893262 sha256=dd444dc99d6ee694eab42418316abb71e8eb0ce086f49c8f88ed89d2e8fb793e\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses, tokenizers, transformers\n",
            "Successfully installed sacremoses-0.0.43 tokenizers-0.10.1 transformers-4.4.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ex5O1eV-Pfct"
      },
      "source": [
        "# 3. Tokenization & Input Formatting\n",
        "\n",
        "In this section, we'll transform our dataset into the format that BERT can be trained on."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-8kEDRvShcU5"
      },
      "source": [
        "## 3.1. BERT Tokenizer"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bWOPOyWghJp2"
      },
      "source": [
        "\n",
        "To feed our text to BERT, it must be split into tokens, and then these tokens must be mapped to their index in the tokenizer vocabulary.\n",
        "\n",
        "The tokenization must be performed by the tokenizer included with BERT--the below cell will download this for us. We'll be using the \"uncased\" version here.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z474sSC6oe7A",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 181,
          "referenced_widgets": [
            "e57c3d7c9e1d4b1f8492f59fd3bd35a8",
            "f67e7b66e057408d8c7c00926c43771b",
            "746cd7b92d2a44d2a474d60f4f58819a",
            "f88c3094aae44297aed0a9bce94214e9",
            "39219d65d33d460fb8569830e3c580e7",
            "e70de0859658432faacaafec55fe597b",
            "449f31c913d24ba4ae8fb313d9b0a567",
            "30452fc30a05406f9fb2f41161a2fe08",
            "962efbff33764c14a5f2ff46c485a3c3",
            "5c370023cad84f709ce595b5deeaed96",
            "4d3163ba1d244ddb8d7bc6513017591a",
            "ac755143edac498698c949dcee6b9624",
            "d34a439b719f4961bfd37148840ffd93",
            "eae8927c9114464ab368f94fb9bc3b9d",
            "ad276274f11f4f8eb05cca79a6c019c9",
            "f00a8f0023bf47dea2d3fcaa2c925ca9",
            "737c993d116b4923b87297a7118edf94",
            "26a68c675e234354b6029728f4f867fc",
            "23aa8345f1ce4b808d1cdb30f559bff2",
            "1bda39694d3045819ac4ff0da845307b",
            "913258ab441d44b6a56222bc067bf072",
            "231ee0da726843a3a69fe701c706ec40",
            "625cc535dfb147d0ac50cd69cfb42279",
            "6ba9bd21d5e14461be1d8d1ad74d5f48"
          ]
        },
        "outputId": "858f0870-38e9-439a-b216-05814ff181b6"
      },
      "source": [
        "from transformers import BertTokenizer\n",
        "\n",
        "# Load the BERT tokenizer.\n",
        "print('Loading BERT tokenizer...')\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-uncased', do_lower_case=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loading BERT tokenizer...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e57c3d7c9e1d4b1f8492f59fd3bd35a8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "962efbff33764c14a5f2ff46c485a3c3",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=28.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "737c993d116b4923b87297a7118edf94",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=466062.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dFzmtleW6KmJ"
      },
      "source": [
        "Let's apply the tokenizer to one sentence just to see the output.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WeNIc4auFUdF"
      },
      "source": [
        "When we actually convert all of our sentences, we'll use the `tokenize.encode` function to handle both steps, rather than calling `tokenize` and `convert_tokens_to_ids` separately. \n",
        "\n",
        "Before we can do that, though, we need to talk about some of BERT's formatting requirements."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l6w8elb-58GJ"
      },
      "source": [
        "## 3.3. Tokenize Dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1M296yz577fV"
      },
      "source": [
        "Just in case there are some longer test sentences, I'll set the maximum length to 64.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tIWAoWL2RK1p"
      },
      "source": [
        "Now we're ready to perform the real tokenization.\n",
        "\n",
        "The `tokenizer.encode_plus` function combines multiple steps for us:\n",
        "\n",
        "1. Split the sentence into tokens.\n",
        "2. Add the special `[CLS]` and `[SEP]` tokens.\n",
        "3. Map the tokens to their IDs.\n",
        "4. Pad or truncate all sentences to the same length.\n",
        "5. Create the attention masks which explicitly differentiate real tokens from `[PAD]` tokens.\n",
        "\n",
        "The first four features are in `tokenizer.encode`, but I'm using `tokenizer.encode_plus` to get the fifth item (attention masks). Documentation is [here](https://huggingface.co/transformers/main_classes/tokenizer.html?highlight=encode_plus#transformers.PreTrainedTokenizer.encode_plus).\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MQkQQbso6OcI"
      },
      "source": [
        "# Load Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tlh18m4lMaVN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4cfe1c3f-71f7-4073-8280-8c9d60328af3"
      },
      "source": [
        "!pip install datasets"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting datasets\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/54/90/43b396481a8298c6010afb93b3c1e71d4ba6f8c10797a7da8eb005e45081/datasets-1.5.0-py3-none-any.whl (192kB)\n",
            "\u001b[K     |████████████████████████████████| 194kB 6.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2.23.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.1.5)\n",
            "Requirement already satisfied: tqdm<4.50.0,>=4.27 in /usr/local/lib/python3.7/dist-packages (from datasets) (4.41.1)\n",
            "Collecting fsspec\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/91/0d/a6bfee0ddf47b254286b9bd574e6f50978c69897647ae15b14230711806e/fsspec-0.8.7-py3-none-any.whl (103kB)\n",
            "\u001b[K     |████████████████████████████████| 112kB 7.2MB/s \n",
            "\u001b[?25hCollecting huggingface-hub<0.1.0\n",
            "  Downloading https://files.pythonhosted.org/packages/af/07/bf95f398e6598202d878332280f36e589512174882536eb20d792532a57d/huggingface_hub-0.0.7-py3-none-any.whl\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from datasets) (3.7.2)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets) (1.19.5)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.3)\n",
            "Requirement already satisfied: pyarrow>=0.17.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (3.0.0)\n",
            "Collecting xxhash\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e7/27/1c0b37c53a7852f1c190ba5039404d27b3ae96a55f48203a74259f8213c9/xxhash-2.0.0-cp37-cp37m-manylinux2010_x86_64.whl (243kB)\n",
            "\u001b[K     |████████████████████████████████| 245kB 9.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.11.1)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2020.12.5)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (1.24.3)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<0.1.0->datasets) (3.0.12)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->datasets) (3.4.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->datasets) (3.7.4.3)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Installing collected packages: fsspec, huggingface-hub, xxhash, datasets\n",
            "Successfully installed datasets-1.5.0 fsspec-0.8.7 huggingface-hub-0.0.7 xxhash-2.0.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 371,
          "referenced_widgets": [
            "1fbb9d0e5d9c48c58680ccea3718110d",
            "f6311cde5f0d4db5841a8d1b1871e08d",
            "836d6676fdc244a7b9b99d201dd71280",
            "8d1814f92f944d67928c95c7525e26f0",
            "6ac31ece46cd4b22963a96d213d2796d",
            "9f644277b15f4fa5a3ae8285325e3b6b",
            "451cfd2d8fcf4bbba82180bddf5afad1",
            "c3d638e64b274f9f827dad41517b3e5f",
            "089845e1098c42fb90a6da033adc1e7b",
            "6ad42df6c502431e8d61b5d6129ba8a8",
            "da7bae5587574cdbbfc69f18aa9e61ec",
            "559bbfb44f6144aa8a2e24ec16e7788d",
            "f0f4e021f3c647a98ad23c1e0a5cd60c",
            "8ee56979b5ae4f069a6800915a47a021",
            "80e99ded59994841939c942a79306938",
            "dec6fb32747540d68d0514cccd1026c3",
            "3d5732ec83a74cf292eaac134d719897",
            "64df45611cd5498eab67e8fd5cbc07d3",
            "f0601b13757c4f0e93bd100ae2de4009",
            "5928c64e567a4cfc8795c28bf11de3d7",
            "e75f82437301484f91d168cf406c0610",
            "0853351f3e8f4cf8a9113815478c92fb",
            "faddabf3315a4c42b46c77aa066667c6",
            "52879479ca034034850d277c6f4fc216",
            "094c8373e1e9462fb1334c0c70bc0e48",
            "36b513532d0f42ff807c985dd2d159ac",
            "71bf6fa3c5844adbafcfe7b8176417f7",
            "de9dd6dae5d842b288ac668f184533ee",
            "7961f9dd530b4de8930d85f6d9ad49e9",
            "f0516e85be9c4f5f88cad5c892cb8530",
            "713cd2ba9824442cb84987c6700e008f",
            "4164df2707fb44eb8a88963a35011474",
            "234d7de0abf6407bbd2d115afe557b7a",
            "8cce77cc40a64c3eb1ed9d6c0a6b6978",
            "c0ddc430cbfd4adcb6202a415225fec1",
            "1d191356dcad4df29fd8a28974ce2cdb",
            "1497d7e17c214ebdb26911ffdef97aee",
            "2287732f01ea4e2a8eab379780cce890",
            "95098afa765d433d8ad4ebfc33c137a5",
            "b77b869cdc304f05a1765bb274131255",
            "298d0ef1ab41422289c5deced2506c58",
            "cec84b95365d4518b076d5fc03fe23a8",
            "8c55a6709ab449288188f50b647fc129",
            "1e2ca6fe8ab24309b5619a2f642efbcf",
            "d0470c10a73747849d4a6906665c504f",
            "9a9b46dbadd1442eac971250a788df30",
            "74e84d085870496baf20afb221d93401",
            "26d7c7e05aa44d9d9d9e0b676d8fe245"
          ]
        },
        "id": "7HuWxZ4kMdf1",
        "outputId": "6ed1a361-70fb-497b-9491-b486bbe105ac"
      },
      "source": [
        "from datasets import load_dataset\n",
        "#languages: ['en', 'de', 'es', 'fr', 'ja', 'ko', 'zh']\n",
        "#splits: ['train', 'test', 'validation']\n",
        "dataset = load_dataset('paws-x', 'en', split='train')\n",
        "print(dataset)\n",
        "val_dataset = load_dataset('paws-x', 'en', split='validation')\n",
        "print(val_dataset)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1fbb9d0e5d9c48c58680ccea3718110d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=2529.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "089845e1098c42fb90a6da033adc1e7b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1648.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Downloading and preparing dataset pawsx/en (download: 28.88 MiB, generated: 12.59 MiB, post-processed: Unknown size, total: 41.47 MiB) to /root/.cache/huggingface/datasets/pawsx/en/1.1.0/19b9c420aa4ca62108be89ef7eca58002d55b175bb1d7e9e1184cb5290bfde20...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3d5732ec83a74cf292eaac134d719897",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=30282057.0, style=ProgressStyle(descrip…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "094c8373e1e9462fb1334c0c70bc0e48",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "234d7de0abf6407bbd2d115afe557b7a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "298d0ef1ab41422289c5deced2506c58",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\rDataset pawsx downloaded and prepared to /root/.cache/huggingface/datasets/pawsx/en/1.1.0/19b9c420aa4ca62108be89ef7eca58002d55b175bb1d7e9e1184cb5290bfde20. Subsequent calls will reuse this data.\n",
            "Dataset({\n",
            "    features: ['id', 'sentence1', 'sentence2', 'label'],\n",
            "    num_rows: 49401\n",
            "})\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Reusing dataset pawsx (/root/.cache/huggingface/datasets/pawsx/en/1.1.0/19b9c420aa4ca62108be89ef7eca58002d55b175bb1d7e9e1184cb5290bfde20)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Dataset({\n",
            "    features: ['id', 'sentence1', 'sentence2', 'label'],\n",
            "    num_rows: 2000\n",
            "})\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dLIbudgfh6F0",
        "outputId": "435d9a58-1602-4e03-d7fa-81855b9047e6"
      },
      "source": [
        "# Print the original sentence.\n",
        "print(' Original: ', dataset[0]['sentence1'])\n",
        "\n",
        "# Print the sentence split into tokens.\n",
        "print('Tokenized: ', tokenizer.tokenize(dataset[0]['sentence1']))\n",
        "x = tokenizer.tokenize(dataset[0]['sentence1'])\n",
        "print(len(x))\n",
        "\n",
        "# Print the sentence mapped to token ids.\n",
        "print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(dataset[0]['sentence1'])))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Original:  In Paris , in October 1560 , he secretly met the English ambassador , Nicolas Throckmorton , asking him for a passport to return to England through Scotland .\n",
            "Tokenized:  ['in', 'paris', ',', 'in', 'october', '1560', ',', 'he', 'secretly', 'met', 'the', 'english', 'ambassador', ',', 'nicolas', 'th', '##rock', '##mo', '##rton', ',', 'asking', 'him', 'for', 'a', 'passport', 'to', 'return', 'to', 'england', 'through', 'scotland', '.']\n",
            "32\n",
            "Token IDs:  [1999, 3000, 1010, 1999, 2255, 29185, 1010, 2002, 10082, 2777, 1996, 2394, 6059, 1010, 9473, 16215, 16901, 5302, 11715, 1010, 4851, 2032, 2005, 1037, 12293, 2000, 2709, 2000, 2563, 2083, 3885, 1012]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6rnHz_eFe73Y"
      },
      "source": [
        "# max_len = 0\n",
        "# ds = val_dataset = load_dataset('paws-x', 'en', split='validation')\n",
        "# # For every sentence...\n",
        "# for sent_dict in ds:\n",
        "#     sent = sent_dict['sentence2']\n",
        "\n",
        "#     # Tokenize the text and add `[CLS]` and `[SEP]` tokens.\n",
        "#     input_ids = tokenizer.encode(sent, add_special_tokens=True)\n",
        "\n",
        "#     # Update the maximum sentence length.\n",
        "#     max_len = max(max_len, len(input_ids))\n",
        "\n",
        "# print('Max sentence length: ', max_len)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Lz87AZgNfWVk"
      },
      "source": [
        "Max Sentence Length: 70 (Train)\n",
        "\n",
        "52 (VAL)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lcnLgqWyi9xK"
      },
      "source": [
        "#Tokenize Training *Set*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2bBdb3pt8LuQ",
        "outputId": "40cd5a4a-a7f7-4422-ee05-f9686e0f6420"
      },
      "source": [
        "# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
        "input_ids = []\n",
        "attention_masks = []\n",
        "label_list = []\n",
        "\n",
        "\n",
        "# For every sentence...\n",
        "for sent_dict in dataset:\n",
        "    sent1 = sent_dict['sentence1']\n",
        "    sent2 = sent_dict['sentence2']\n",
        "    label_list.append(sent_dict['label'])\n",
        "    # `encode_plus` will:\n",
        "    #   (1) Tokenize the sentence.\n",
        "    #   (2) Prepend the `[CLS]` token to the start.\n",
        "    #   (3) Append the `[SEP]` token to the end.\n",
        "    #   (4) Map tokens to their IDs.\n",
        "    #   (5) Pad or truncate the sentence to `max_length`\n",
        "    #   (6) Create attention masks for [PAD] tokens.\n",
        "    encoded_dict = tokenizer.encode_plus(\n",
        "                        sent1, sent2,                      # Sentence to encode.\n",
        "                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "                        max_length = 128,           # Pad & truncate all sentences.\n",
        "                        pad_to_max_length = True,\n",
        "                        return_attention_mask = True,   # Construct attn. masks.\n",
        "                        return_tensors = 'pt',     # Return pytorch tensors.\n",
        "                        \n",
        "                   )\n",
        "    \n",
        "    # Add the encoded sentence to the list.    \n",
        "    input_ids.append(encoded_dict['input_ids'])\n",
        "    \n",
        "    # And its attention mask (simply differentiates padding from non-padding).\n",
        "    attention_masks.append(encoded_dict['attention_mask'])\n",
        "\n",
        "\n",
        "# Convert the lists into tensors.\n",
        "input_ids = torch.cat(input_ids, dim=0)\n",
        "attention_masks = torch.cat(attention_masks, dim=0)\n",
        "labels = torch.tensor(label_list)\n",
        "\n",
        "\n",
        "# Print sentence 0, now as a list of IDs.\n",
        "print('Original: ', dataset[0]['sentence1'])\n",
        "print('Token IDs:', input_ids[0])\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2074: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Original:  In Paris , in October 1560 , he secretly met the English ambassador , Nicolas Throckmorton , asking him for a passport to return to England through Scotland .\n",
            "Token IDs: tensor([  101,  1999,  3000,  1010,  1999,  2255, 29185,  1010,  2002, 10082,\n",
            "         2777,  1996,  2394,  6059,  1010,  9473, 16215, 16901,  5302, 11715,\n",
            "         1010,  4851,  2032,  2005,  1037, 12293,  2000,  2709,  2000,  2563,\n",
            "         2083,  3885,  1012,   102,  1999,  2255, 29185,  1010,  2002, 10082,\n",
            "         2777,  2007,  1996,  2394,  6059,  1010,  9473, 16215, 16901,  5302,\n",
            "        11715,  1010,  1999,  3000,  1010,  1998,  2356,  2032,  2005,  1037,\n",
            "        12293,  2000,  2709,  2000,  3885,  2083,  2563,  1012,   102,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x6F78o8mgTDv"
      },
      "source": [
        "[CLS] 101\n",
        "\n",
        "[SEP] 102"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_142bEhqjCQ6"
      },
      "source": [
        "#Tokenize Validation set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tE7hoqHKhajv",
        "outputId": "4632c60b-d4c2-4107-d3e5-ab73837be3a8"
      },
      "source": [
        "# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
        "val_input_ids = []\n",
        "val_attention_masks = []\n",
        "val_label_list = []\n",
        "\n",
        "\n",
        "\n",
        "# For every sentence...\n",
        "for sent_dict in val_dataset:\n",
        "    sent1 = sent_dict['sentence1']\n",
        "    sent2 = sent_dict['sentence2']\n",
        "    val_label_list.append(sent_dict['label'])\n",
        "    # `encode_plus` will:\n",
        "    #   (1) Tokenize the sentence.\n",
        "    #   (2) Prepend the `[CLS]` token to the start.\n",
        "    #   (3) Append the `[SEP]` token to the end.\n",
        "    #   (4) Map tokens to their IDs.\n",
        "    #   (5) Pad or truncate the sentence to `max_length`\n",
        "    #   (6) Create attention masks for [PAD] tokens.\n",
        "    encoded_dict_val = tokenizer.encode_plus(\n",
        "                        sent1, sent2,                      # Sentence to encode.\n",
        "                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "                        max_length = 128,           # Pad & truncate all sentences.\n",
        "                        pad_to_max_length = True,\n",
        "                        return_attention_mask = True,   # Construct attn. masks.\n",
        "                        return_tensors = 'pt',     # Return pytorch tensors.\n",
        "                        \n",
        "                   )\n",
        "    \n",
        "    # Add the encoded sentence to the list.    \n",
        "    val_input_ids.append(encoded_dict_val['input_ids'])\n",
        "    \n",
        "    # And its attention mask (simply differentiates padding from non-padding).\n",
        "    val_attention_masks.append(encoded_dict_val['attention_mask'])\n",
        "\n",
        "\n",
        "# Convert the lists into tensors.\n",
        "val_input_ids = torch.cat(val_input_ids, dim=0)\n",
        "val_attention_masks = torch.cat(val_attention_masks, dim=0)\n",
        "val_labels = torch.tensor(val_label_list)\n",
        "\n",
        "\n",
        "# Print sentence 0, now as a list of IDs.\n",
        "print('Original: ', val_dataset[0]['sentence1'])\n",
        "print('Token IDs:', val_input_ids[0])\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2074: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Original:  From the merger of the Four Rivers Council and the Audubon Council , the Shawnee Trails Council was born .\n",
            "Token IDs: tensor([  101,  2013,  1996,  7660,  1997,  1996,  2176,  5485,  2473,  1998,\n",
            "         1996,  8740,  8566, 11735,  2473,  1010,  1996, 13218,  4402,  9612,\n",
            "         2473,  2001,  2141,  1012,   102, 13218,  4402,  9612,  2473,  2001,\n",
            "         2719,  2013,  1996,  7660,  1997,  1996,  2176,  5485,  2473,  1998,\n",
            "         1996,  8740,  8566, 11735,  2473,  1012,   102,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aRp4O7D295d_"
      },
      "source": [
        "## 3.4. Training & Validation Split\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qu0ao7p8rb06"
      },
      "source": [
        "Divide up our training set to use 90% for training and 10% for validation."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xIHTKQ4Jgf5c"
      },
      "source": [
        "from torch.utils.data import TensorDataset, random_split\n",
        "\n",
        "# Combine the training inputs into a TensorDataset.\n",
        "train_dataset = TensorDataset(input_ids, attention_masks, labels)\n",
        "\n",
        "dev_dataset = TensorDataset(val_input_ids, val_attention_masks, val_labels)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dD9i6Z2pG-sN"
      },
      "source": [
        "We'll also create an iterator for our dataset using the torch DataLoader class. This helps save on memory during training because, unlike a for loop, with an iterator the entire dataset does not need to be loaded into memory."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XGUqOCtgqGhP"
      },
      "source": [
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
        "\n",
        "# The DataLoader needs to know our batch size for training, so we specify it \n",
        "# here. For fine-tuning BERT on a specific task, the authors recommend a batch \n",
        "# size of 16 or 32.\n",
        "batch_size = 32\n",
        "val_batch_size = 16\n",
        "\n",
        "# Create the DataLoaders for our training and validation sets.\n",
        "# We'll take training samples in random order. \n",
        "train_dataloader = DataLoader(\n",
        "            train_dataset,  # The training samples.\n",
        "            sampler = RandomSampler(train_dataset), # Select batches randomly\n",
        "            batch_size = batch_size # Trains with this batch size.\n",
        "        )\n",
        "\n",
        "# For validation the order doesn't matter, so we'll just read them sequentially.\n",
        "validation_dataloader = DataLoader(\n",
        "            dev_dataset, # The validation samples.\n",
        "            sampler = SequentialSampler(dev_dataset), # Pull out batches sequentially.\n",
        "            batch_size = val_batch_size # Evaluate with this batch size.\n",
        "        )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8bwa6Rts-02-"
      },
      "source": [
        "# 4. Train Our Classification Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3xYQ3iLO08SX"
      },
      "source": [
        "Now that our input data is properly formatted, it's time to fine tune the BERT model. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D6TKgyUzPIQc"
      },
      "source": [
        "## 4.1. BertForSequenceClassification"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1sjzRT1V0zwm"
      },
      "source": [
        "For this task, we first want to modify the pre-trained BERT model to give outputs for classification, and then we want to continue training the model on our dataset until that the entire model, end-to-end, is well-suited for our task. \n",
        "\n",
        "Thankfully, the huggingface pytorch implementation includes a set of interfaces designed for a variety of NLP tasks. Though these interfaces are all built on top of a trained BERT model, each has different top layers and output types designed to accomodate their specific NLP task.  \n",
        "\n",
        "Here is the current list of classes provided for fine-tuning:\n",
        "* BertModel\n",
        "* BertForPreTraining\n",
        "* BertForMaskedLM\n",
        "* BertForNextSentencePrediction\n",
        "* **BertForSequenceClassification** - The one we'll use.\n",
        "* BertForTokenClassification\n",
        "* BertForQuestionAnswering\n",
        "\n",
        "The documentation for these can be found under [here](https://huggingface.co/transformers/v2.2.0/model_doc/bert.html)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BXYitPoE-cjH"
      },
      "source": [
        "\n",
        "\n",
        "We'll be using [BertForSequenceClassification](https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#bertforsequenceclassification). This is the normal BERT model with an added single linear layer on top for classification that we will use as a sentence classifier. As we feed input data, the entire pre-trained BERT model and the additional untrained classification layer is trained on our specific task. \n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WnQW9E-bBCRt"
      },
      "source": [
        "OK, let's load BERT! There are a few different pre-trained BERT models available. \"bert-base-uncased\" means the version that has only lowercase letters (\"uncased\") and is the smaller version of the two (\"base\" vs \"large\").\n",
        "\n",
        "The documentation for `from_pretrained` can be found [here](https://huggingface.co/transformers/v2.2.0/main_classes/model.html#transformers.PreTrainedModel.from_pretrained), with the additional parameters defined [here](https://huggingface.co/transformers/v2.2.0/main_classes/configuration.html#transformers.PretrainedConfig)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gFsCTp_mporB",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "af3531e3c8124027bfb237c86d5c094f",
            "50070c08115d4103ab27aea687535f23",
            "2646ad7872834cd083a1b6c092942e01",
            "0f548e00e6f94e1f968d845e2c18b084",
            "9b00175550304222a33e9ea2e9f0d7fa",
            "7ffe02e27f8a4e7c832604e97b173c4c",
            "27abed83dd53475f850f1e56d9168cf1",
            "f9ff3c83c055436d8e1be8e8f9b4bfc1",
            "3c16ccfb58e040618a6ef57eb7e820e3",
            "702ab50ea5fd4f058cd0d7b414d12098",
            "321be65546654aa9ae7943bb89ef5713",
            "b4f09bfb735a4091a15532252e85f71d",
            "3fad0e0f98454367a6bab2949c7d4837",
            "743b4b8b64ce41d898e62be6374ce5b7",
            "a76fa1f419104733881d53fe380bae02",
            "b157dea0e639449e964f73b294b01720"
          ]
        },
        "outputId": "0e1de57a-629d-4b67-9ec7-a17146de5b90"
      },
      "source": [
        "from transformers import BertForSequenceClassification, AdamW, BertConfig\n",
        "\n",
        "# Load BertForSequenceClassification, the pretrained BERT model with a single \n",
        "# linear classification layer on top. \n",
        "model = BertForSequenceClassification.from_pretrained(\n",
        "    \"bert-base-multilingual-uncased\", # Use the 12-layer BERT model, with an uncased vocab.\n",
        "    num_labels = 2, # The number of output labels--2 for binary classification.\n",
        "                    # You can increase this for multi-class tasks.   \n",
        "    output_attentions = False, # Whether the model returns attentions weights.\n",
        "    output_hidden_states = False, # Whether the model returns all hidden-states.\n",
        ")\n",
        "\n",
        "# Tell pytorch to run this model on the GPU.\n",
        "model.cuda()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "af3531e3c8124027bfb237c86d5c094f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=625.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3c16ccfb58e040618a6ef57eb7e820e3",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=672271273.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-multilingual-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-multilingual-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertForSequenceClassification(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(105879, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e0Jv6c7-HHDW"
      },
      "source": [
        "Just for curiosity's sake, we can browse all of the model's parameters by name here.\n",
        "\n",
        "In the below cell, I've printed out the names and dimensions of the weights for:\n",
        "\n",
        "1. The embedding layer.\n",
        "2. The first of the twelve transformers.\n",
        "3. The output layer.\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8PIiVlDYCtSq",
        "outputId": "e2952842-9389-49e9-bd85-0ef237fb49b7"
      },
      "source": [
        "# Get all of the model's parameters as a list of tuples.\n",
        "params = list(model.named_parameters())\n",
        "\n",
        "print('The BERT model has {:} different named parameters.\\n'.format(len(params)))\n",
        "\n",
        "print('==== Embedding Layer ====\\n')\n",
        "\n",
        "for p in params[0:5]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "\n",
        "print('\\n==== First Transformer ====\\n')\n",
        "\n",
        "for p in params[5:21]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "\n",
        "print('\\n==== Output Layer ====\\n')\n",
        "\n",
        "for p in params[-4:]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The BERT model has 201 different named parameters.\n",
            "\n",
            "==== Embedding Layer ====\n",
            "\n",
            "bert.embeddings.word_embeddings.weight                  (105879, 768)\n",
            "bert.embeddings.position_embeddings.weight                (512, 768)\n",
            "bert.embeddings.token_type_embeddings.weight                (2, 768)\n",
            "bert.embeddings.LayerNorm.weight                              (768,)\n",
            "bert.embeddings.LayerNorm.bias                                (768,)\n",
            "\n",
            "==== First Transformer ====\n",
            "\n",
            "bert.encoder.layer.0.attention.self.query.weight          (768, 768)\n",
            "bert.encoder.layer.0.attention.self.query.bias                (768,)\n",
            "bert.encoder.layer.0.attention.self.key.weight            (768, 768)\n",
            "bert.encoder.layer.0.attention.self.key.bias                  (768,)\n",
            "bert.encoder.layer.0.attention.self.value.weight          (768, 768)\n",
            "bert.encoder.layer.0.attention.self.value.bias                (768,)\n",
            "bert.encoder.layer.0.attention.output.dense.weight        (768, 768)\n",
            "bert.encoder.layer.0.attention.output.dense.bias              (768,)\n",
            "bert.encoder.layer.0.attention.output.LayerNorm.weight        (768,)\n",
            "bert.encoder.layer.0.attention.output.LayerNorm.bias          (768,)\n",
            "bert.encoder.layer.0.intermediate.dense.weight           (3072, 768)\n",
            "bert.encoder.layer.0.intermediate.dense.bias                 (3072,)\n",
            "bert.encoder.layer.0.output.dense.weight                 (768, 3072)\n",
            "bert.encoder.layer.0.output.dense.bias                        (768,)\n",
            "bert.encoder.layer.0.output.LayerNorm.weight                  (768,)\n",
            "bert.encoder.layer.0.output.LayerNorm.bias                    (768,)\n",
            "\n",
            "==== Output Layer ====\n",
            "\n",
            "bert.pooler.dense.weight                                  (768, 768)\n",
            "bert.pooler.dense.bias                                        (768,)\n",
            "classifier.weight                                           (2, 768)\n",
            "classifier.bias                                                 (2,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qRWT-D4U_Pvx"
      },
      "source": [
        "## 4.2. Optimizer & Learning Rate Scheduler"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8o-VEBobKwHk"
      },
      "source": [
        "Now that we have our model loaded we need to grab the training hyperparameters from within the stored model.\n",
        "\n",
        "For the purposes of fine-tuning, the authors recommend choosing from the following values (from Appendix A.3 of the [BERT paper](https://arxiv.org/pdf/1810.04805.pdf)):\n",
        "\n",
        ">- **Batch size:** 16, 32  \n",
        "- **Learning rate (Adam):** 5e-5, 3e-5, 2e-5  \n",
        "- **Number of epochs:** 2, 3, 4 \n",
        "\n",
        "We chose:\n",
        "* Batch size: 32 (set when creating our DataLoaders)\n",
        "* Learning rate: 2e-5\n",
        "* Epochs: 4 (we'll see that this is probably too many...)\n",
        "\n",
        "The epsilon parameter `eps = 1e-8` is \"a very small number to prevent any division by zero in the implementation\" (from [here](https://machinelearningmastery.com/adam-optimization-algorithm-for-deep-learning/)).\n",
        "\n",
        "You can find the creation of the AdamW optimizer in `run_glue.py` [here](https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L109)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GLs72DuMODJO"
      },
      "source": [
        "# Note: AdamW is a class from the huggingface library (as opposed to pytorch) \n",
        "# I believe the 'W' stands for 'Weight Decay fix\"\n",
        "optimizer = AdamW(model.parameters(),\n",
        "                  lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
        "                  eps = 1e-8 # args.adam_epsilon  - default is 1e-8.\n",
        "                )\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-p0upAhhRiIx"
      },
      "source": [
        "from transformers import get_linear_schedule_with_warmup\n",
        "\n",
        "# Number of training epochs. The BERT authors recommend between 2 and 4. \n",
        "# We chose to run for 4, but we'll see later that this may be over-fitting the\n",
        "# training data.\n",
        "epochs = 4\n",
        "\n",
        "# Total number of training steps is [number of batches] x [number of epochs]. \n",
        "# (Note that this is not the same as the number of training samples).\n",
        "total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "# Create the learning rate scheduler.\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
        "                                            num_warmup_steps = 0, # Default value in run_glue.py\n",
        "                                            num_training_steps = total_steps)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RqfmWwUR_Sox"
      },
      "source": [
        "## 4.3. Training Loop"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_QXZhFb4LnV5"
      },
      "source": [
        "Below is our training loop. There's a lot going on, but fundamentally for each pass in our loop we have a trianing phase and a validation phase. \n",
        "\n",
        "> *Thank you to [Stas Bekman](https://ca.linkedin.com/in/stasbekman) for contributing the insights and code for using validation loss to detect over-fitting!*\n",
        "\n",
        "**Training:**\n",
        "- Unpack our data inputs and labels\n",
        "- Load data onto the GPU for acceleration\n",
        "- Clear out the gradients calculated in the previous pass. \n",
        "    - In pytorch the gradients accumulate by default (useful for things like RNNs) unless you explicitly clear them out.\n",
        "- Forward pass (feed input data through the network)\n",
        "- Backward pass (backpropagation)\n",
        "- Tell the network to update parameters with optimizer.step()\n",
        "- Track variables for monitoring progress\n",
        "\n",
        "**Evalution:**\n",
        "- Unpack our data inputs and labels\n",
        "- Load data onto the GPU for acceleration\n",
        "- Forward pass (feed input data through the network)\n",
        "- Compute loss on our validation data and track variables for monitoring progress\n",
        "\n",
        "Pytorch hides all of the detailed calculations from us, but we've commented the code to point out which of the above steps are happening on each line. \n",
        "\n",
        "> *PyTorch also has some [beginner tutorials](https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html#sphx-glr-beginner-blitz-cifar10-tutorial-py) which you may also find helpful.*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pE5B99H5H2-W"
      },
      "source": [
        "Define a helper function for calculating accuracy."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9cQNvaZ9bnyy"
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "# Function to calculate the accuracy of our predictions vs labels\n",
        "def flat_accuracy(preds, labels):\n",
        "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    return np.sum(pred_flat == labels_flat) / len(labels_flat)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KNhRtWPXH9C3"
      },
      "source": [
        "Helper function for formatting elapsed times as `hh:mm:ss`\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gpt6tR83keZD"
      },
      "source": [
        "import time\n",
        "import datetime\n",
        "\n",
        "def format_time(elapsed):\n",
        "    '''\n",
        "    Takes a time in seconds and returns a string hh:mm:ss\n",
        "    '''\n",
        "    # Round to the nearest second.\n",
        "    elapsed_rounded = int(round((elapsed)))\n",
        "    \n",
        "    # Format as hh:mm:ss\n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cfNIhN19te3N"
      },
      "source": [
        "We're ready to kick off the training!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6J-FYdx6nFE_",
        "outputId": "d5f38d56-bcc9-4900-c204-afb01a0d41ef"
      },
      "source": [
        " import random\n",
        "import numpy as np\n",
        "\n",
        "# This training code is based on the `run_glue.py` script here:\n",
        "# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n",
        "\n",
        "# Set the seed value all over the place to make this reproducible.\n",
        "seed_val = 42\n",
        "\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)\n",
        "\n",
        "# We'll store a number of quantities such as training and validation loss, \n",
        "# validation accuracy, and timings.\n",
        "training_stats = []\n",
        "\n",
        "# Measure the total training time for the whole run.\n",
        "total_t0 = time.time()\n",
        "\n",
        "# For each epoch...\n",
        "for epoch_i in range(0, epochs):\n",
        "    \n",
        "    # ========================================\n",
        "    #               Training\n",
        "    # ========================================\n",
        "    \n",
        "    # Perform one full pass over the training set.\n",
        "\n",
        "    print(\"\")\n",
        "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
        "    print('Training...')\n",
        "\n",
        "    # Measure how long the training epoch takes.\n",
        "    t0 = time.time()\n",
        "\n",
        "    # Reset the total loss for this epoch.\n",
        "    total_train_loss = 0\n",
        "\n",
        "    # Put the model into training mode. Don't be mislead--the call to \n",
        "    # `train` just changes the *mode*, it doesn't *perform* the training.\n",
        "    # `dropout` and `batchnorm` layers behave differently during training\n",
        "    # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)\n",
        "    model.train()\n",
        "\n",
        "    # For each batch of training data...\n",
        "    for step, batch in enumerate(train_dataloader):\n",
        "\n",
        "        # Progress update every 40 batches.\n",
        "        if step % 40 == 0 and not step == 0:\n",
        "            # Calculate elapsed time in minutes.\n",
        "            elapsed = format_time(time.time() - t0)\n",
        "            \n",
        "            # Report progress.\n",
        "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
        "\n",
        "        # Unpack this training batch from our dataloader. \n",
        "        #\n",
        "        # As we unpack the batch, we'll also copy each tensor to the GPU using the \n",
        "        # `to` method.\n",
        "        #\n",
        "        # `batch` contains three pytorch tensors:\n",
        "        #   [0]: input ids \n",
        "        #   [1]: attention masks\n",
        "        #   [2]: labels \n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "\n",
        "        # Always clear any previously calculated gradients before performing a\n",
        "        # backward pass. PyTorch doesn't do this automatically because \n",
        "        # accumulating the gradients is \"convenient while training RNNs\". \n",
        "        # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)\n",
        "        model.zero_grad()        \n",
        "\n",
        "        # Perform a forward pass (evaluate the model on this training batch).\n",
        "        # In PyTorch, calling `model` will in turn call the model's `forward` \n",
        "        # function and pass down the arguments. The `forward` function is \n",
        "        # documented here: \n",
        "        # https://huggingface.co/transformers/model_doc/bert.html#bertforsequenceclassification\n",
        "        # The results are returned in a results object, documented here:\n",
        "        # https://huggingface.co/transformers/main_classes/output.html#transformers.modeling_outputs.SequenceClassifierOutput\n",
        "        # Specifically, we'll get the loss (because we provided labels) and the\n",
        "        # \"logits\"--the model outputs prior to activation.\n",
        "        result = model(b_input_ids, \n",
        "                       token_type_ids=None, \n",
        "                       attention_mask=b_input_mask, \n",
        "                       labels=b_labels,\n",
        "                       return_dict=True)\n",
        "\n",
        "        loss = result.loss\n",
        "        logits = result.logits\n",
        "\n",
        "        # Accumulate the training loss over all of the batches so that we can\n",
        "        # calculate the average loss at the end. `loss` is a Tensor containing a\n",
        "        # single value; the `.item()` function just returns the Python value \n",
        "        # from the tensor.\n",
        "        total_train_loss += loss.item()\n",
        "\n",
        "        # Perform a backward pass to calculate the gradients.\n",
        "        loss.backward()\n",
        "\n",
        "        # Clip the norm of the gradients to 1.0.\n",
        "        # This is to help prevent the \"exploding gradients\" problem.\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "        # Update parameters and take a step using the computed gradient.\n",
        "        # The optimizer dictates the \"update rule\"--how the parameters are\n",
        "        # modified based on their gradients, the learning rate, etc.\n",
        "        optimizer.step()\n",
        "\n",
        "        # Update the learning rate.\n",
        "        scheduler.step()\n",
        "\n",
        "    # Calculate the average loss over all of the batches.\n",
        "    avg_train_loss = total_train_loss / len(train_dataloader)            \n",
        "    \n",
        "    # Measure how long this epoch took.\n",
        "    training_time = format_time(time.time() - t0)\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "    print(\"  Training epcoh took: {:}\".format(training_time))\n",
        "        \n",
        "    # ========================================\n",
        "    #               Validation\n",
        "    # ========================================\n",
        "    # After the completion of each training epoch, measure our performance on\n",
        "    # our validation set.\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"Running Validation...\")\n",
        "\n",
        "    t0 = time.time()\n",
        "\n",
        "    # Put the model in evaluation mode--the dropout layers behave differently\n",
        "    # during evaluation.\n",
        "    model.eval()\n",
        "\n",
        "    # Tracking variables \n",
        "    total_eval_accuracy = 0\n",
        "    total_eval_loss = 0\n",
        "    nb_eval_steps = 0\n",
        "\n",
        "    # Evaluate data for one epoch\n",
        "    for batch in validation_dataloader:\n",
        "        \n",
        "        # Unpack this training batch from our dataloader. \n",
        "        #\n",
        "        # As we unpack the batch, we'll also copy each tensor to the GPU using \n",
        "        # the `to` method.\n",
        "        #\n",
        "        # `batch` contains three pytorch tensors:\n",
        "        #   [0]: input ids \n",
        "        #   [1]: attention masks\n",
        "        #   [2]: labels \n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "        \n",
        "        # Tell pytorch not to bother with constructing the compute graph during\n",
        "        # the forward pass, since this is only needed for backprop (training).\n",
        "        with torch.no_grad():        \n",
        "\n",
        "            # Forward pass, calculate logit predictions.\n",
        "            # token_type_ids is the same as the \"segment ids\", which \n",
        "            # differentiates sentence 1 and 2 in 2-sentence tasks.\n",
        "            result = model(b_input_ids, \n",
        "                           token_type_ids=None, \n",
        "                           attention_mask=b_input_mask,\n",
        "                           labels=b_labels,\n",
        "                           return_dict=True)\n",
        "\n",
        "        # Get the loss and \"logits\" output by the model. The \"logits\" are the \n",
        "        # output values prior to applying an activation function like the \n",
        "        # softmax.\n",
        "        loss = result.loss\n",
        "        logits = result.logits\n",
        "            \n",
        "        # Accumulate the validation loss.\n",
        "        total_eval_loss += loss.item()\n",
        "\n",
        "        # Move logits and labels to CPU\n",
        "        logits = logits.detach().cpu().numpy()\n",
        "        label_ids = b_labels.to('cpu').numpy()\n",
        "\n",
        "        # Calculate the accuracy for this batch of test sentences, and\n",
        "        # accumulate it over all batches.\n",
        "        total_eval_accuracy += flat_accuracy(logits, label_ids)\n",
        "        \n",
        "\n",
        "    # Report the final accuracy for this validation run.\n",
        "    avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)\n",
        "    print(\"  Accuracy: {0:.2f}\".format(avg_val_accuracy))\n",
        "\n",
        "    # Calculate the average loss over all of the batches.\n",
        "    avg_val_loss = total_eval_loss / len(validation_dataloader)\n",
        "    \n",
        "    # Measure how long the validation run took.\n",
        "    validation_time = format_time(time.time() - t0)\n",
        "    \n",
        "    print(\"  Validation Loss: {0:.2f}\".format(avg_val_loss))\n",
        "    print(\"  Validation took: {:}\".format(validation_time))\n",
        "\n",
        "    # Record all statistics from this epoch.\n",
        "    training_stats.append(\n",
        "        {\n",
        "            'epoch': epoch_i + 1,\n",
        "            'Training Loss': avg_train_loss,\n",
        "            'Valid. Loss': avg_val_loss,\n",
        "            'Valid. Accur.': avg_val_accuracy,\n",
        "            'Training Time': training_time,\n",
        "            'Validation Time': validation_time\n",
        "        }\n",
        "    )\n",
        "\n",
        "print(\"\")\n",
        "print(\"Training complete!\")\n",
        "\n",
        "print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "======== Epoch 1 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of  1,544.    Elapsed: 0:00:31.\n",
            "  Batch    80  of  1,544.    Elapsed: 0:01:03.\n",
            "  Batch   120  of  1,544.    Elapsed: 0:01:34.\n",
            "  Batch   160  of  1,544.    Elapsed: 0:02:06.\n",
            "  Batch   200  of  1,544.    Elapsed: 0:02:39.\n",
            "  Batch   240  of  1,544.    Elapsed: 0:03:11.\n",
            "  Batch   280  of  1,544.    Elapsed: 0:03:43.\n",
            "  Batch   320  of  1,544.    Elapsed: 0:04:15.\n",
            "  Batch   360  of  1,544.    Elapsed: 0:04:47.\n",
            "  Batch   400  of  1,544.    Elapsed: 0:05:20.\n",
            "  Batch   440  of  1,544.    Elapsed: 0:05:52.\n",
            "  Batch   480  of  1,544.    Elapsed: 0:06:24.\n",
            "  Batch   520  of  1,544.    Elapsed: 0:06:56.\n",
            "  Batch   560  of  1,544.    Elapsed: 0:07:28.\n",
            "  Batch   600  of  1,544.    Elapsed: 0:08:01.\n",
            "  Batch   640  of  1,544.    Elapsed: 0:08:33.\n",
            "  Batch   680  of  1,544.    Elapsed: 0:09:05.\n",
            "  Batch   720  of  1,544.    Elapsed: 0:09:37.\n",
            "  Batch   760  of  1,544.    Elapsed: 0:10:10.\n",
            "  Batch   800  of  1,544.    Elapsed: 0:10:42.\n",
            "  Batch   840  of  1,544.    Elapsed: 0:11:14.\n",
            "  Batch   880  of  1,544.    Elapsed: 0:11:46.\n",
            "  Batch   920  of  1,544.    Elapsed: 0:12:18.\n",
            "  Batch   960  of  1,544.    Elapsed: 0:12:51.\n",
            "  Batch 1,000  of  1,544.    Elapsed: 0:13:23.\n",
            "  Batch 1,040  of  1,544.    Elapsed: 0:13:55.\n",
            "  Batch 1,080  of  1,544.    Elapsed: 0:14:27.\n",
            "  Batch 1,120  of  1,544.    Elapsed: 0:14:59.\n",
            "  Batch 1,160  of  1,544.    Elapsed: 0:15:32.\n",
            "  Batch 1,200  of  1,544.    Elapsed: 0:16:04.\n",
            "  Batch 1,240  of  1,544.    Elapsed: 0:16:36.\n",
            "  Batch 1,280  of  1,544.    Elapsed: 0:17:08.\n",
            "  Batch 1,320  of  1,544.    Elapsed: 0:17:41.\n",
            "  Batch 1,360  of  1,544.    Elapsed: 0:18:13.\n",
            "  Batch 1,400  of  1,544.    Elapsed: 0:18:45.\n",
            "  Batch 1,440  of  1,544.    Elapsed: 0:19:17.\n",
            "  Batch 1,480  of  1,544.    Elapsed: 0:19:50.\n",
            "  Batch 1,520  of  1,544.    Elapsed: 0:20:22.\n",
            "\n",
            "  Average training loss: 0.46\n",
            "  Training epcoh took: 0:20:41\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.86\n",
            "  Validation Loss: 0.39\n",
            "  Validation took: 0:00:16\n",
            "\n",
            "======== Epoch 2 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of  1,544.    Elapsed: 0:00:32.\n",
            "  Batch    80  of  1,544.    Elapsed: 0:01:04.\n",
            "  Batch   120  of  1,544.    Elapsed: 0:01:37.\n",
            "  Batch   160  of  1,544.    Elapsed: 0:02:09.\n",
            "  Batch   200  of  1,544.    Elapsed: 0:02:41.\n",
            "  Batch   240  of  1,544.    Elapsed: 0:03:13.\n",
            "  Batch   280  of  1,544.    Elapsed: 0:03:46.\n",
            "  Batch   320  of  1,544.    Elapsed: 0:04:18.\n",
            "  Batch   360  of  1,544.    Elapsed: 0:04:50.\n",
            "  Batch   400  of  1,544.    Elapsed: 0:05:22.\n",
            "  Batch   440  of  1,544.    Elapsed: 0:05:54.\n",
            "  Batch   480  of  1,544.    Elapsed: 0:06:27.\n",
            "  Batch   520  of  1,544.    Elapsed: 0:06:59.\n",
            "  Batch   560  of  1,544.    Elapsed: 0:07:31.\n",
            "  Batch   600  of  1,544.    Elapsed: 0:08:04.\n",
            "  Batch   640  of  1,544.    Elapsed: 0:08:36.\n",
            "  Batch   680  of  1,544.    Elapsed: 0:09:08.\n",
            "  Batch   720  of  1,544.    Elapsed: 0:09:40.\n",
            "  Batch   760  of  1,544.    Elapsed: 0:10:12.\n",
            "  Batch   800  of  1,544.    Elapsed: 0:10:45.\n",
            "  Batch   840  of  1,544.    Elapsed: 0:11:17.\n",
            "  Batch   880  of  1,544.    Elapsed: 0:11:49.\n",
            "  Batch   920  of  1,544.    Elapsed: 0:12:21.\n",
            "  Batch   960  of  1,544.    Elapsed: 0:12:54.\n",
            "  Batch 1,000  of  1,544.    Elapsed: 0:13:26.\n",
            "  Batch 1,040  of  1,544.    Elapsed: 0:13:58.\n",
            "  Batch 1,080  of  1,544.    Elapsed: 0:14:30.\n",
            "  Batch 1,120  of  1,544.    Elapsed: 0:15:02.\n",
            "  Batch 1,160  of  1,544.    Elapsed: 0:15:35.\n",
            "  Batch 1,200  of  1,544.    Elapsed: 0:16:07.\n",
            "  Batch 1,240  of  1,544.    Elapsed: 0:16:39.\n",
            "  Batch 1,280  of  1,544.    Elapsed: 0:17:12.\n",
            "  Batch 1,320  of  1,544.    Elapsed: 0:17:44.\n",
            "  Batch 1,360  of  1,544.    Elapsed: 0:18:16.\n",
            "  Batch 1,400  of  1,544.    Elapsed: 0:18:48.\n",
            "  Batch 1,440  of  1,544.    Elapsed: 0:19:21.\n",
            "  Batch 1,480  of  1,544.    Elapsed: 0:19:53.\n",
            "  Batch 1,520  of  1,544.    Elapsed: 0:20:25.\n",
            "\n",
            "  Average training loss: 0.26\n",
            "  Training epcoh took: 0:20:44\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.86\n",
            "  Validation Loss: 0.41\n",
            "  Validation took: 0:00:16\n",
            "\n",
            "======== Epoch 3 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of  1,544.    Elapsed: 0:00:32.\n",
            "  Batch    80  of  1,544.    Elapsed: 0:01:05.\n",
            "  Batch   120  of  1,544.    Elapsed: 0:01:37.\n",
            "  Batch   160  of  1,544.    Elapsed: 0:02:09.\n",
            "  Batch   200  of  1,544.    Elapsed: 0:02:41.\n",
            "  Batch   240  of  1,544.    Elapsed: 0:03:14.\n",
            "  Batch   280  of  1,544.    Elapsed: 0:03:46.\n",
            "  Batch   320  of  1,544.    Elapsed: 0:04:18.\n",
            "  Batch   360  of  1,544.    Elapsed: 0:04:51.\n",
            "  Batch   400  of  1,544.    Elapsed: 0:05:23.\n",
            "  Batch   440  of  1,544.    Elapsed: 0:05:55.\n",
            "  Batch   480  of  1,544.    Elapsed: 0:06:27.\n",
            "  Batch   520  of  1,544.    Elapsed: 0:07:00.\n",
            "  Batch   560  of  1,544.    Elapsed: 0:07:32.\n",
            "  Batch   600  of  1,544.    Elapsed: 0:08:04.\n",
            "  Batch   640  of  1,544.    Elapsed: 0:08:37.\n",
            "  Batch   680  of  1,544.    Elapsed: 0:09:09.\n",
            "  Batch   720  of  1,544.    Elapsed: 0:09:41.\n",
            "  Batch   760  of  1,544.    Elapsed: 0:10:14.\n",
            "  Batch   800  of  1,544.    Elapsed: 0:10:46.\n",
            "  Batch   840  of  1,544.    Elapsed: 0:11:18.\n",
            "  Batch   880  of  1,544.    Elapsed: 0:11:50.\n",
            "  Batch   920  of  1,544.    Elapsed: 0:12:23.\n",
            "  Batch   960  of  1,544.    Elapsed: 0:12:55.\n",
            "  Batch 1,000  of  1,544.    Elapsed: 0:13:27.\n",
            "  Batch 1,040  of  1,544.    Elapsed: 0:14:00.\n",
            "  Batch 1,080  of  1,544.    Elapsed: 0:14:32.\n",
            "  Batch 1,120  of  1,544.    Elapsed: 0:15:04.\n",
            "  Batch 1,160  of  1,544.    Elapsed: 0:15:37.\n",
            "  Batch 1,200  of  1,544.    Elapsed: 0:16:09.\n",
            "  Batch 1,240  of  1,544.    Elapsed: 0:16:41.\n",
            "  Batch 1,280  of  1,544.    Elapsed: 0:17:14.\n",
            "  Batch 1,320  of  1,544.    Elapsed: 0:17:46.\n",
            "  Batch 1,360  of  1,544.    Elapsed: 0:18:18.\n",
            "  Batch 1,400  of  1,544.    Elapsed: 0:18:51.\n",
            "  Batch 1,440  of  1,544.    Elapsed: 0:19:23.\n",
            "  Batch 1,480  of  1,544.    Elapsed: 0:19:55.\n",
            "  Batch 1,520  of  1,544.    Elapsed: 0:20:28.\n",
            "\n",
            "  Average training loss: 0.18\n",
            "  Training epcoh took: 0:20:47\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.87\n",
            "  Validation Loss: 0.40\n",
            "  Validation took: 0:00:16\n",
            "\n",
            "======== Epoch 4 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of  1,544.    Elapsed: 0:00:32.\n",
            "  Batch    80  of  1,544.    Elapsed: 0:01:04.\n",
            "  Batch   120  of  1,544.    Elapsed: 0:01:37.\n",
            "  Batch   160  of  1,544.    Elapsed: 0:02:09.\n",
            "  Batch   200  of  1,544.    Elapsed: 0:02:42.\n",
            "  Batch   240  of  1,544.    Elapsed: 0:03:14.\n",
            "  Batch   280  of  1,544.    Elapsed: 0:03:46.\n",
            "  Batch   320  of  1,544.    Elapsed: 0:04:19.\n",
            "  Batch   360  of  1,544.    Elapsed: 0:04:51.\n",
            "  Batch   400  of  1,544.    Elapsed: 0:05:23.\n",
            "  Batch   440  of  1,544.    Elapsed: 0:05:56.\n",
            "  Batch   480  of  1,544.    Elapsed: 0:06:28.\n",
            "  Batch   520  of  1,544.    Elapsed: 0:07:01.\n",
            "  Batch   560  of  1,544.    Elapsed: 0:07:33.\n",
            "  Batch   600  of  1,544.    Elapsed: 0:08:05.\n",
            "  Batch   640  of  1,544.    Elapsed: 0:08:37.\n",
            "  Batch   680  of  1,544.    Elapsed: 0:09:10.\n",
            "  Batch   720  of  1,544.    Elapsed: 0:09:42.\n",
            "  Batch   760  of  1,544.    Elapsed: 0:10:15.\n",
            "  Batch   800  of  1,544.    Elapsed: 0:10:47.\n",
            "  Batch   840  of  1,544.    Elapsed: 0:11:19.\n",
            "  Batch   880  of  1,544.    Elapsed: 0:11:52.\n",
            "  Batch   920  of  1,544.    Elapsed: 0:12:24.\n",
            "  Batch   960  of  1,544.    Elapsed: 0:12:56.\n",
            "  Batch 1,000  of  1,544.    Elapsed: 0:13:29.\n",
            "  Batch 1,040  of  1,544.    Elapsed: 0:14:01.\n",
            "  Batch 1,080  of  1,544.    Elapsed: 0:14:33.\n",
            "  Batch 1,120  of  1,544.    Elapsed: 0:15:06.\n",
            "  Batch 1,160  of  1,544.    Elapsed: 0:15:38.\n",
            "  Batch 1,200  of  1,544.    Elapsed: 0:16:10.\n",
            "  Batch 1,240  of  1,544.    Elapsed: 0:16:43.\n",
            "  Batch 1,280  of  1,544.    Elapsed: 0:17:15.\n",
            "  Batch 1,320  of  1,544.    Elapsed: 0:17:47.\n",
            "  Batch 1,360  of  1,544.    Elapsed: 0:18:20.\n",
            "  Batch 1,400  of  1,544.    Elapsed: 0:18:52.\n",
            "  Batch 1,440  of  1,544.    Elapsed: 0:19:25.\n",
            "  Batch 1,480  of  1,544.    Elapsed: 0:19:57.\n",
            "  Batch 1,520  of  1,544.    Elapsed: 0:20:29.\n",
            "\n",
            "  Average training loss: 0.13\n",
            "  Training epcoh took: 0:20:49\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.88\n",
            "  Validation Loss: 0.46\n",
            "  Validation took: 0:00:16\n",
            "\n",
            "Training complete!\n",
            "Total training took 1:24:06 (h:mm:ss)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oWvUapjRSuNy"
      },
      "source": [
        "## 4.4. Plot Training Results"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "6O_NbXFGMukX",
        "outputId": "c043ce44-2296-49b0-a24c-9c2f94b16989"
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "# Display floats with two decimal places.\n",
        "pd.set_option('precision', 2)\n",
        "\n",
        "# Create a DataFrame from our training statistics.\n",
        "df_stats = pd.DataFrame(data=training_stats)\n",
        "\n",
        "# Use the 'epoch' as the row index.\n",
        "df_stats = df_stats.set_index('epoch')\n",
        "\n",
        "# A hack to force the column headers to wrap.\n",
        "#df = df.style.set_table_styles([dict(selector=\"th\",props=[('max-width', '70px')])])\n",
        "\n",
        "# Display the table.\n",
        "df_stats"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Valid. Loss</th>\n",
              "      <th>Valid. Accur.</th>\n",
              "      <th>Training Time</th>\n",
              "      <th>Validation Time</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>epoch</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.46</td>\n",
              "      <td>0.39</td>\n",
              "      <td>0.86</td>\n",
              "      <td>0:20:41</td>\n",
              "      <td>0:00:16</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.26</td>\n",
              "      <td>0.41</td>\n",
              "      <td>0.86</td>\n",
              "      <td>0:20:44</td>\n",
              "      <td>0:00:16</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.18</td>\n",
              "      <td>0.40</td>\n",
              "      <td>0.87</td>\n",
              "      <td>0:20:47</td>\n",
              "      <td>0:00:16</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.13</td>\n",
              "      <td>0.46</td>\n",
              "      <td>0.88</td>\n",
              "      <td>0:20:49</td>\n",
              "      <td>0:00:16</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       Training Loss  Valid. Loss  Valid. Accur. Training Time Validation Time\n",
              "epoch                                                                         \n",
              "1               0.46         0.39           0.86       0:20:41         0:00:16\n",
              "2               0.26         0.41           0.86       0:20:44         0:00:16\n",
              "3               0.18         0.40           0.87       0:20:47         0:00:16\n",
              "4               0.13         0.46           0.88       0:20:49         0:00:16"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 427
        },
        "id": "68xreA9JAmG5",
        "outputId": "88324ded-707f-40b2-87cf-b9b2061b88c8"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "% matplotlib inline\n",
        "\n",
        "import seaborn as sns\n",
        "\n",
        "# Use plot styling from seaborn.\n",
        "sns.set(style='darkgrid')\n",
        "\n",
        "# Increase the plot size and font size.\n",
        "sns.set(font_scale=1.5)\n",
        "plt.rcParams[\"figure.figsize\"] = (12,6)\n",
        "\n",
        "# Plot the learning curve.\n",
        "plt.plot(df_stats['Training Loss'], 'b-o', label=\"Training\")\n",
        "plt.plot(df_stats['Valid. Loss'], 'g-o', label=\"Validation\")\n",
        "\n",
        "# Label the plot.\n",
        "plt.title(\"Training & Validation Loss\")\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.legend()\n",
        "plt.xticks([1, 2, 3, 4])\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvAAAAGaCAYAAABpIXfbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeVyU5f4//tcMDDPsm4CsisgABwVxNy13JcUlxTQ9kmVmix/9Wp7SY3XK8/P0yTRLzVN6stIDGiKmllmmZvWp5JAdcQEX1JRQQWDYZ4aZuX9/AKPDsMwoMDPwej4ePXKue7vukct5z8X7et8iQRAEEBERERGRTRBbugNERERERGQ6BvBERERERDaEATwRERERkQ1hAE9EREREZEMYwBMRERER2RAG8ERERERENoQBPBF1enl5eYiIiMDGjRvv+RzLly9HREREK/aq42rq/Y6IiMDy5ctNOsfGjRsRERGBvLy8Vu9feno6IiIicOLEiVY/NxFRa7C3dAeIiBoyJxA+cuQIgoKC2rA3tqeqqgoffPABDh48iIKCAnh5eaFfv3547rnnEBYWZtI5Fi9ejK+//hqff/45oqKiGt1HEASMHj0aZWVl+PHHHyGTyVrzNtrUiRMnkJGRgccffxxubm6W7o6RvLw8jB49GnPmzMFrr71m6e4QkZVhAE9EVmfNmjUGr3/99Vd89tlnmDlzJvr162ewzcvL676vFxgYiKysLNjZ2d3zOf7+97/jjTfeuO++tIZXXnkFX375JRISEjBw4EAUFhbi6NGjOHXqlMkBfGJiIr7++mvs2bMHr7zySqP7/PLLL/jjjz8wc+bMVgnes7KyIBa3zy+GMzIysGnTJjzyyCNGAfyUKVMwceJESCSSdukLEZG5GMATkdWZMmWKwWutVovPPvsMffr0MdrWUEVFBVxcXMy6nkgkglQqNbufd7OWYK+6uhqHDh3CsGHDsG7dOn37okWLoFarTT7PsGHD4O/vjwMHDuCll16Cg4OD0T7p6ekAaoP91nC/fwetxc7O7r6+zBERtTXmwBORzRo1ahTmzp2Lc+fOYf78+ejXrx8mT54MoDaQX79+PWbMmIFBgwahV69eGDt2LNauXYvq6mqD8zSWk31327FjxzB9+nT07t0bw4YNw1tvvQWNRmNwjsZy4OvbysvL8be//Q1DhgxB7969MWvWLJw6dcrofkpKSrBixQoMGjQIcXFxSEpKwrlz5zB37lyMGjXKpPdEJBJBJBI1+oWisSC8KWKxGI888ggUCgWOHj1qtL2iogLffPMN5HI5YmJizHq/m9JYDrxOp8OHH36IUaNGoXfv3khISMD+/fsbPT43Nxevv/46Jk6ciLi4OMTGxmLatGnYvXu3wX7Lly/Hpk2bAACjR49GRESEwd9/UznwxcXFeOONNzB8+HD06tULw4cPxxtvvIGSkhKD/eqP//nnn/HRRx9hzJgx6NWrF8aPH4+9e/ea9F6YIycnB88//zwGDRqE3r17Y8KECdi6dSu0Wq3Bfjdu3MCKFSswcuRI9OrVC0OGDMGsWbMM+qTT6fDJJ59g0qRJiIuLQ9++fTF+/Hj89a9/RU1NTav3nYjuDWfgicim5efn4/HHH0d8fDzGjRuHqqoqAMCtW7eQlpaGcePGISEhAfb29sjIyMC//vUvZGdn46OPPjLp/MePH0dKSgpmzZqF6dOn48iRI9i2bRvc3d3xzDPPmHSO+fPnw8vLC88//zwUCgU+/vhjPP300zhy5Ij+twVqtRpPPPEEsrOzMW3aNPTu3Rvnz5/HE088AXd3d5PfD5lMhqlTp2LPnj344osvkJCQYPKxDU2bNg3//Oc/kZ6ejvj4eINtX375JZRKJaZPnw6g9d7vht58801s374dAwYMwLx581BUVIRVq1YhODjYaN+MjAxkZmZixIgRCAoK0v824pVXXkFxcTEWLlwIAJg5cyYqKipw+PBhrFixAp6engCaX3tRXl6Oxx57DL///jumT5+OP/3pT8jOzsbOnTvxyy+/YPfu3Ua/+Vm/fj2USiVmzpwJBwcH7Ny5E8uXL0dISIhRKti9On36NObOnQt7e3vMmTMHXbp0wbFjx7B27Vrk5OTofwuj0WjwxBNP4NatW5g9eza6d++OiooKnD9/HpmZmXjkkUcAAP/85z+xYcMGjBw5ErNmzYKdnR3y8vJw9OhRqNVqq/lNE1GnJxARWbk9e/YIcrlc2LNnj0H7yJEjBblcLqSmphodo1KpBLVabdS+fv16QS6XC6dOndK3Xb9+XZDL5cKGDRuM2mJjY4Xr16/r23U6nTBx4kRh6NChBud9+eWXBblc3mjb3/72N4P2gwcPCnK5XNi5c6e+7d///rcgl8uFzZs3G+xb3z5y5Eije2lMeXm5sGDBAqFXr17Cn/70J+HLL7806bimJCUlCVFRUcKtW7cM2h999FEhOjpaKCoqEgTh/t9vQRAEuVwuvPzyy/rXubm5QkREhJCUlCRoNBp9+5kzZ4SIiAhBLpcb/N1UVlYaXV+r1Qp//vOfhb59+xr0b8OGDUbH16v/efvll1/0be+8844gl8uFf//73wb71v/9rF+/3uj4KVOmCCqVSt9+8+ZNITo6Wli6dKnRNRuqf4/eeOONZvebOXOmEBUVJWRnZ+vbdDqdsHjxYkEulws//fSTIAiCkJ2dLcjlcmHLli3Nnm/q1KnCww8/3GL/iMiymEJDRDbNw8MD06ZNM2p3cHDQzxZqNBqUlpaiuLgYDzzwAAA0msLSmNGjRxtUuRGJRBg0aBAKCwtRWVlp0jnmzZtn8Hrw4MEAgN9//13fduzYMdjZ2SEpKclg3xkzZsDV1dWk6+h0OixZsgQ5OTn46quv8NBDD2HZsmU4cOCAwX6vvvoqoqOjTcqJT0xMhFarxeeff65vy83NxX//+1+MGjVKv4i4td7vux05cgSCIOCJJ54wyEmPjo7G0KFDjfZ3cnLS/1mlUqGkpAQKhQJDhw5FRUUFLl++bHYf6h0+fBheXl6YOXOmQfvMmTPh5eWFb7/91uiY2bNnG6Qt+fn5ITQ0FFevXr3nftytqKgIv/32G0aNGoXIyEh9u0gkwrPPPqvvNwD9z9CJEydQVFTU5DldXFxw69YtZGZmtkofiahtMIWGiGxacHBwkwsOk5OTsWvXLly6dAk6nc5gW2lpqcnnb8jDwwMAoFAo4OzsbPY56lM2FAqFvi0vLw++vr5G53NwcEBQUBDKyspavM6RI0fw448/4u2330ZQUBDee+89LFq0CC+99BI0Go0+TeL8+fPo3bu3STnx48aNg5ubG9LT0/H0008DAPbs2QMA+vSZeq3xft/t+vXrAIAePXoYbQsLC8OPP/5o0FZZWYlNmzbhq6++wo0bN4yOMeU9bEpeXh569eoFe3vDj017e3t0794d586dMzqmqZ+dP/7445770bBPANCzZ0+jbT169IBYLNa/h4GBgXjmmWewZcsWDBs2DFFRURg8eDDi4+MRExOjP+6FF17A888/jzlz5sDX1xcDBw7EiBEjMH78eLPWUBBR22IAT0Q2zdHRsdH2jz/+GP/7v/+LYcOGISkpCb6+vpBIJLh16xaWL18OQRBMOn9z1Uju9xymHm+q+kWXAwYMAFAb/G/atAnPPvssVqxYAY1Gg8jISJw6dQqrV6826ZxSqRQJCQlISUnByZMnERsbi/3796Nr16548MEH9fu11vt9P1588UV89913ePTRRzFgwAB4eHjAzs4Ox48fxyeffGL0paKttVdJTFMtXboUiYmJ+O6775CZmYm0tDR89NFHeOqpp/CXv/wFABAXF4fDhw/jxx9/xIkTJ3DixAl88cUX+Oc//4mUlBT9l1cisiwG8ETUIe3btw+BgYHYunWrQSD1/fffW7BXTQsMDMTPP/+MyspKg1n4mpoa5OXlmfSwofr7/OOPP+Dv7w+gNojfvHkznnnmGbz66qsIDAyEXC7H1KlTTe5bYmIiUlJSkJ6ejtLSUhQWFuKZZ54xeF/b4v2un8G+fPkyQkJCDLbl5uYavC4rK8N3332HKVOmYNWqVQbbfvrpJ6Nzi0Qis/ty5coVaDQag1l4jUaDq1evNjrb3tbqU7suXbpktO3y5cvQ6XRG/QoODsbcuXMxd+5cqFQqzJ8/H//617/w5JNPwtvbGwDg7OyM8ePHY/z48QBqf7OyatUqpKWl4amnnmrjuyIiU1jX9AARUSsRi8UQiUQGM78ajQZbt261YK+aNmrUKGi1Wmzfvt2gPTU1FeXl5SadY/jw4QBqq5/cnd8ulUrxzjvvwM3NDXl5eRg/frxRKkhzoqOjERUVhYMHDyI5ORkikcio9ntbvN+jRo2CSCTCxx9/bFAS8ezZs0ZBef2XhoYz/QUFBUZlJIE7+fKmpvaMGTMGxcXFRudKTU1FcXExxowZY9J5WpO3tzfi4uJw7NgxXLhwQd8uCAK2bNkCABg7diyA2io6DctASqVSfXpS/ftQXFxsdJ3o6GiDfYjI8jgDT0QdUnx8PNatW4cFCxZg7NixqKiowBdffGFW4NqeZsyYgV27duHdd9/FtWvX9GUkDx06hG7duhnVnW/M0KFDkZiYiLS0NEycOBFTpkxB165dcf36dezbtw9AbTD2/vvvIywsDA8//LDJ/UtMTMTf//53/PDDDxg4cKDRzG5bvN9hYWGYM2cO/v3vf+Pxxx/HuHHjUFRUhOTkZERGRhrknbu4uGDo0KHYv38/ZDIZevfujT/++AOfffYZgoKCDNYbAEBsbCwAYO3atZg0aRKkUinCw8Mhl8sb7ctTTz2FQ4cOYdWqVTh37hyioqKQnZ2NtLQ0hIaGttnM9JkzZ7B582ajdnt7ezz99NNYuXIl5s6dizlz5mD27Nnw8fHBsWPH8OOPPyIhIQFDhgwBUJte9eqrr2LcuHEIDQ2Fs7Mzzpw5g7S0NMTGxuoD+QkTJqBPnz6IiYmBr68vCgsLkZqaColEgokTJ7bJPRKR+azzk4yI6D7Nnz8fgiAgLS0Nq1evho+PDx5++GFMnz4dEyZMsHT3jDg4OODTTz/FmjVrcOTIEXz11VeIiYnBJ598gpUrV0KpVJp0ntWrV2PgwIHYtWsXPvroI9TU1CAwMBDx8fF48skn4eDggJkzZ+Ivf/kLXF1dMWzYMJPOO2nSJKxZswYqlcpo8SrQdu/3ypUr0aVLF6SmpmLNmjXo3r07XnvtNfz+++9GC0fffvttrFu3DkePHsXevXvRvXt3LF26FPb29lixYoXBvv369cOyZcuwa9cuvPrqq9BoNFi0aFGTAbyrqyt27tyJDRs24OjRo0hPT4e3tzdmzZqF//mf/zH76b+mOnXqVKMVfBwcHPD000+jd+/e2LVrFzZs2ICdO3eiqqoKwcHBWLZsGZ588kn9/hERERg7diwyMjJw4MAB6HQ6+Pv7Y+HChQb7Pfnkkzh+/Dh27NiB8vJyeHt7IzY2FgsXLjSodENEliUS2mNlERER3ROtVovBgwcjJibmnh+GREREHQtz4ImIrERjs+y7du1CWVlZo3XPiYioc2IKDRGRlXjllVegVqsRFxcHBwcH/Pbbb/jiiy/QrVs3PProo5buHhERWQmm0BARWYnPP/8cycnJuHr1KqqqquDt7Y3hw4djyZIl6NKli6W7R0REVoIBPBERERGRDWEOPBERERGRDWEAT0RERERkQ7iI1UwlJZXQ6do/68jb2wVFRRXtfl0iW8OxQmQajhUi01hirIjFInh6Oje5nQG8mXQ6wSIBfP21iahlHCtEpuFYITKNtY0VptAQEREREdkQBvBERERERDaEATwRERERkQ1hAE9EREREZEMYwBMRERER2RAG8ERERERENoQBPBERERGRDWEAT0RERERkQxjAExERERHZED6JlYiIiIiogYybJ7E/9xAUKgU8pB6YHBaPgV37WrpbABjAExEREREZyLh5Eik5e1CjqwEAlKgUSMnZAwBWEcQzgLdyP5+9ifTjuSguU8HLTYppw8MwJLqrpbtFRERE1CFpdBqkX/xCH7zXq9HVYH/uIQbw1Lyfz97Ep1/lQK3RAQCKylT49KscAGAQT0RERNQKBEHAzaoC5BRfRE7xBVxQXIZaq2503xKVop171zgG8FYs/XiuPnivp9bokH48lwE8ERER0T0qU5fXBey1/5WqywAAvo5dMLhrP5wsyEJFTaXRcZ5Sj/buaqMsGsCr1Wq899572LdvH8rKyhAZGYmlS5diyJAhZp1nwYIF+P7775GUlISVK1cabIuIiGj0mNdffx2PPfbYPfe9PRSVqcxqJyIiIiJjaq0alxRXagP2kov4o+IGAMDZ3glyr56I8gxHpFc4vB29AACh7t0McuABQCKWYHJYvEX635BFA/jly5fjm2++QVJSErp164a9e/diwYIF2LFjB+Li4kw6x3fffYfMzMxm9xk2bBgmT55s0BYbG3vP/W4v3m7SRoN1V0eJBXpDREREZBt0gg55Ffn6Gfbc0qvQ6DSwF9mhh3t3TO4Rj0ivcAS7BkIsMq6qXp/nzio0DWRlZeHLL7/EihUrMG/ePADA1KlTkZCQgLVr1yI5ObnFc6jVarz55puYP38+Nm7c2OR+PXr0wJQpU1qr6+1m2vAwgxx4ABABKK+uwZFf8zC6X5DlOkdERERkRYqVJfqA/XzJJX0KTIBzVzwUOASRXnL09AiF1M7BpPMN7NoXA7v2hY+PKwoLy9uy62azWAB/6NAhSCQSzJgxQ98mlUqRmJiI9evXo6CgAL6+vs2eY/v27VAqlS0G8ACgVCohEokglUpbpf/toT7P/e4qNJOHhuK3i7eRfPgCikqVSBwZBrFIZOGeEhEREbWvao0SF0py69JiLqCg6jYAwM3BFX/yjkBkXVqMu9TNwj1tfRYL4LOzsxEaGgpnZ2eD9piYGAiCgOzs7GYD+MLCQmzevBmvvfYaHB0dm71WWloaduzYAUEQIJfLsXjxYowdO7ZV7qOtDYnuiiHRXQ2+/Q3t7Y+Uby/gUMY13C5TYkFCFCT2dhbuKREREVHb0eq0+L38OrLrZtmvll2DTtDBQSxBT88eeDBgMCK95PB39oOog09uWiyALywshJ+fn1G7j48PAKCgoKDZ49955x2Ehoa2mBoTFxeHCRMmICgoCDdu3MD27duxaNEirFu3DgkJCfd+AxYkFoswZ6wcXdwdkXrsEhQVKiyeHgMX5sYTERFRByEIAgqqb+vTYi6U5EKpVUIEEUJcgzA2ZAQivcIR6t4NEnHnKqxosbtVKpWQSIwDzvoUF5Wq6UorWVlZ+Pzzz7Fjx44Wv2Ht2rXL4PUjjzyChIQEvP3225g4caLZ39C8vV3M2r81+fi4GryemxCN7kEeWL/zJN5KOYnXFwxBV2/nJo4m6jwajhUiahzHClmbMlUFztzKQdbNbGTdysHtqmIAgI+zN4Z264+YrpHo5RsBV2n7xmPWNlYsFsDLZDLU1NQYtdcH7k3lqguCgNWrV2PcuHHo37+/2dd1cnLCrFmzsG7dOly+fBlhYWFmHV9UVAGdTjD7uverqQUUkYFueHFmH2zck4UX3j2OJYmx6BHQ8XK9iExljYuNiKwRxwpZgxqdBpcVV5FTUvsQpevl+RAgwNFeBrlnT4wOGo5Ir3D4OHrrJ12VZQKUaL+fXUuMFbFY1OykscUCeB8fn0bTZAoLCwGgyfz3w4cPIysrC0uXLkVeXp7BtoqKCuTl5aFLly6QyWRNXtvf3x8AUFpaeq/dtyryYA/8dW4/rE89hTUpJ7FwSjTiwn0s3S0iIiIiA4IgIL/yJrKLLyCn+CIuKa6gRlcDsUiMULdumBg6FpFe4QhxDYKdmOv7mmKxAD4yMhI7duxAZWWlwULWU6dO6bc3Jj8/HzqdDo8//rjRtvT0dKSnp2Pr1q146KGHmrz29evXAQBeXl73cwtWxd/bGSuT+mND2ilsSj+N2WPkLDNJREREFqdQld556mnJRZSrKwAAfk6+eCBgIKK8whHu0QMy+6YnX8mQxQL4+Ph4bNu2Dbt379bXgVer1UhPT0ffvn31C1zz8/NRXV2tT3UZNWoUgoKMA9Pnn38eI0eORGJiIqKjowEAxcXFRkF6SUkJUlJSEBQUhO7du7fdDVqAu7MDXnqsLz7cf5ZlJomIiMgiVFo1Lpbk1qXFXMSNylsAABeJMyK9wvXlHT1lHhbuqe2yWAAfGxuL+Ph4rF27FoWFhQgJCcHevXuRn5+PN998U7/fyy+/jIyMDJw/fx4AEBISgpCQkEbPGRwcjDFjxuhfJycn48iRIxgxYgQCAgJw69YtfPbZZyguLsb777/ftjdoIVIHOyya1ptlJomIiKhd6AQdrpXn6WfZL5f+Dq2ghURsjzD3UAzq2g+RXnIEunRt9KmnZD6L1txZs2YN3n33Xezbtw+lpaWIiIjAli1b0K9fv1Y5f1xcHE6ePIndu3ejtLQUTk5O6NOnDxYuXNhq17BGLDNJREREbel2dZG+HvuFkkuo0lQDAIJcAjAyeBgivcIR5h4KBzvGHm1BJAhC+5dUsWHWVoWmJRnZt/CvL7Lh7S7D0kdj4evR/EOviGwdK2sQmYZjhcxRVVONCyWXkF2XFnO7uggA4CF1R6RXOKI8wxHhFQ5XB8uV224rrEJD7W5glB88XKTYuCcLq7dnsswkERERtUij0+BK6TV9HvvvZdchQIDUzgFyzzCMDKqdZfdz8unwTz21RpyBN5OtzcDXu1FUifWpp1BWqWaZSerQOKtIZBqOFbqbIAi4VVVQlxZzARcUl6HWqiGCCN3dgmsXn3rJEeoW0unKO3IGniyGZSaJiIjobuXqCoPyjgpV7fNxfBy96xaehkPuEQYnCdNvrQ0D+E6EZSaJiIg6L7W2BrmKK8guqX2I0h8VNwAATvaOiKjLY4/0Coe3Y8d5Tk5HxQC+k2GZSSIios5BJ+iQV5Gvn2XPLb0KjU4DO5Ederh3w6Qe8YjyCkewayDLO9oYBvCdEMtMEhERdUwlSoU+j/18ySVU1FQCAAKcu+KhwCGI9ApHT48ekNo5WLindD8YwHdSIpEI8YNC4OUmxb++yMbqHb+yzCQREZGNqdYoDZ56equqEADg5uCKP3lH6J966i5lBbqOhAF8J8cyk0RERLZDq9Pi9/Lr+ocoXS27Bp2gg0QsQbhHDwwNGIQoLzn8nf1Y3rEDYxlJM9lqGcmWsMwkdRQsjUdkGo4V2yAIAgqrb+vz2M+X5EKpVUIEEYJdA2sfouQVjlD37pCIOS/bFlhGkqwWy0wSERFZh4qaSpwvvqQv71isLAEAeMk80c8vBpFecsg9w+AicbZwT8lSGMCTHstMEhERtb8anQZXSq/qF59eL8+HAAEyOxkiPMMwNmQ4Ir3C4ePYhWkxBIABPDXAMpNERERtSxAE5Ffe1KfFXFJchlpXA7FIjFC3EEwIHYNILzm6uQZ1uqeekmkYwJMRlpkkIiJqXaWqMuQUX0R28UWcL7mIMnVtTrWfkw+GBAxEVF15R0d7mYV7SraAATw1imUmiYiI7p1KqzYo73ij8hYAwEXijAjPnoj0kiPKKxyeMg8L95RsEQN4ahbLTBIREbVMJ+hwvfwPfR775dLfoRW0sBfbo6d7KAZ17YdIr3AEuvjzqad031hG0kwdtYxkS1hmkmyFpccKka3gWLl/t6uLkVN8oa684yVUaaoBAEEuAYj0qn2AUph7KBzsmIJqy1hGkmwWy0wSEVFnV1VTjQsll5BdlxZzu7oIAOAhdUdMl2h90O7q0HTgRdQaGMCTyVhmkoiIOhOtTosrZdf0s+xXy65DgACpnQPCPcIwImgoorzC4efky/KO1K4YwJNZWGaSiIg6KkEQcKuqoC6P/SIuKnKh0qohggjd3YIR330UIr3k6O4WDHs+9ZQsiD99ZDaWmSQioo6iXF2hr8eeU3IRClUpAKCLozcGdO2LKM9wyD3D4CRxsnBPie5gAE/3hGUmiYjIFqm1NcgtvaIP2vMq8gEATvaOdeUdwxHpJUcXRy8L95SoaQzg6b6wzCQREVkznaDDHxU37jz1tPQKNDoN7ER26OHeDZN6xCPKKxzBroEs70g2g2UkzdRZy0i2hGUmyVpY+1ghshYdeayUKBX6euznSy6hoqYSAODv7Fc7w+5Z+9RTmb3Uwj0lW8AyktRhscwkERFZilKjxEXFZf3i01tVBQAAVwcXRHlFIMorHBFePeEhdbdwT4laBwN4ajUsM0lERO1Bq9Pi9/I8fXnHK2XXoBN0kIglCPfogaEBAxHpFY4A564s70gdEgN4alUsM0mWknHzJPbnHoJCpYCH1AOTw+IxsGtfS3eLiFqBIAgorL6tz2O/oMhFtUYJEUQIdg3AmJDhiPQMRw+P7pCwvCN1Avwpp1bHMpPU3jJunkRKzh7U6GoAACUqBVJy9gAAg3giG1VZU4XzJZf0s+xFyhIAgJfME3E+MYj0CkeEZ0+4ODhbuKdE7Y+LWM3ERazmyci+hX99kQ1vdxnLTNJ9EQQBSq0KpaoylKrKoFCVolRd++f/y8/QB+93k4jtEdMlGjJ7GWT2UjjaySC1l0JmV/taZiet3WYnhaO9DFI7KWT2UlaioE7B2j5XanQaXCm9qs9jv17+BwQIkNnJEOEZVlfeMRw+jl2YFkPtiotYqdNhmUkyhVpbgzJ1GRR1wXmpqhQKdZk+WC9VlUGhLoNaqzY61tFe1mjwDtQGBNfL/0C1VgmVRgV1E/s15GDnAEc76V3Bvqwu2G88+Ddo17fJ4CCWMNAgaoIgCLhReQvZdTPslxSXodbVQCwSo7tbCB4OHYMor3B0cw2GnZhpmER34wy8mTgDf29YZrJz0uq0KFOX62fKa2fO6/58V1ulpsroWInYHu4ObnCXusND6gb3uv88HO782c3BDTJ7KV75v3+gRKUwOoen1AP/39C/GvRHpVWhWqOCSquCUqtEtUYFpUZZ+1qjRLVWBZWmdptSo9IH/8q67UqtCtUaJXSCrsX7F0HUfPBv9EWg4f/vHMPHtlNrs8TnSqmqTP/E0/PFF1Gqrr2+n5OPvrxjuGcYHO1l7dovouZY4ww8A5gq71oAACAASURBVHgzMYC/d6WVamxIO4WrN8tZZtLG6QQdKmuq6oLxUuMAve51uboCAgzHi1gkhpuDa20wLnWvC9INg3MPqRsc7R1Nnr1umAMPABKxBLMjp7dJDrwgCNDoNHVBfX2wr2zw+q6gv7G2un1VWpXRe9QYe7F9bVBfF+BL7aRwvGvGX1qXImT8Wlq3750/M0WIgPb5XFFp1bikuKxffJpfeRMA4CJxrnvqqRyRXj3hJfNs034Q3Q8G8B0AA/j7o1Jr8eH+s/jvpduIHxjCMpNWpjbPXGk4U16XvqJPbVGVoUxdDq2gNTreVeKiD8bdHdwMZs7rA3YXiXObBJC2WoVGJ+ig1qpbCP7vDvqb+r+qyVSihupThGT6vH9Zg9eGwb/MXgbHu/4ss6t9LWGKkE1ri88VnaDD9fI/9A9RulL6OzSCFvZie4S5d9fnsQe5BPCLJNkMBvANqNVqvPfee9i3bx/KysoQGRmJpUuXYsiQIWadZ8GCBfj++++RlJSElStXGm3fvXs3tm3bhry8PAQEBCApKQlz5sy5pz4zgL9/Op2AlG8v4OjJP9A/0pdlJtuJWltzV+pKaaOpLApVaaN54o72stpUFocGAblBOourVaR5dKSxYq67U4SMZ/yNvxiotLUpQkqDNKLaNnNThO5eBNzymgGZURqRNfzsdDatNVaKqouRU3wR2SUXcaH4kj4lLtDFH5Fe4YjylCPMIxQOdqxERrbJGgN4i/6LuXz5cnzzzTdISkpCt27dsHfvXixYsAA7duxAXFycSef47rvvkJmZ2eT2Xbt24W9/+xvi4+PxxBNPIDMzE6tWrYJKpcKTTz7ZWrdCZmCZydZVn2euaCQYvztAr9JUGx0rEdvDvS6NJdg1EL26RN0VmLvrg3OpnYMF7ozMZSe2g5PYCU4Sp/s6z90pQtV1wX/9OoDqu4L/xtYMVGmqUaJUGPx2wBT6FKFmg/+W1hAwRag9VNVU44Iity4t5gIKq4sAAB5Sd/Tu8qfa8o5ePeHm4GrhnhJ1XBabgc/KysKMGTOwYsUKzJs3DwCgUqmQkJAAX19fJCcnt3gOtVqNSZMmYdKkSdi4caPRDLxSqcTw4cPRr18/bN68Wd++bNkyHD16FMePH4erq3n/wHAGvnWxzGTTjPLMG6Sy1L+uUFc2mmdukFtel9JydyqLu4MbHO1lHS4FoqOOFVtlmCJ01zqBZhcMN76GoEanMemaUjuHu4L9xoL+u74kNPKFQdpJUoRMHStanRZXyq7p67FfLbsOAQIc7Bwg9+hRl8cejq5Ovh36/aLOizPwdzl06BAkEglmzJihb5NKpUhMTMT69etRUFAAX1/fZs+xfft2KJVKzJ8/Hxs3bjTafuLECSgUCsyePdugfc6cOThw4AC+//57TJw4sXVuiO5JZywzKQgCqjVK/cx4w5nyu1NbGktjcJW46HPLQ9yC6nLN3Q1SW9oqz5zIXGKRuC5wlgFS9/s6l1anNf4i0HABcROLicvVFQbHmpIiJBaJaxcEN1gEbNqaAcNttlQGURAE3KoqrKsWcwEXSy5DqVVBBBG6uQVjfPdRiPQMR6h7CFOfiCzEYiMvOzsboaGhcHY2fIJaTEwMBEFAdnZ2swF8YWEhNm/ejNdeew2Ojo3P2p47dw4A0KtXL4P26OhoiMVinDt3jgG8FZAHe+Cvc/thfeoprEk5adNlJtVatVFuecMAvVRV1kSeuaM+fcXPM6xBnnltKUU3B1ebCgSIWpOd2A7OYic4t0KKUI1OY3LwX9+uqksRKlYq7qwZMDFFSCK2bzL41/8WwOhZAsa/QZDaOdz3l/PGFnxHeclxvi6PPaf4IhSqUgBAF0dv9O8ahyjPcMg9w+47PYuIWofFAvjCwkL4+fkZtfv41AZuBQUFzR7/zjvvIDQ0FFOmTGn2Gg4ODvDw8DBor29r6RrUfvy9nbEyqT82pJ3CpvTTVldm8k6eecNUFsMqLdWN5plL7syYuwbBo4t7g0ot7nCXusKBeeZE7UIkEsHBTgIHO8l952nrBB1UWrU+oK9fQKzSqFDd6ILhO69LVaW4WXUnjcj8FKEWHirWyALiiyW5OHDlG33FohKVAtvPfaZPw3O0d6wr7xiOKK9wdHH0vq/3h4jahsUCeKVSCYnEeNGiVCoFUJsP35SsrCx8/vnn2LFjR7P5dk1do/46zV2jKc3lI7U1H5+OvSDIxwdY8z8PYW3yr0g+fAFVNTrMm/gniMVtl1OpE3QoU1WgpLoUxdUKlFQrUFxdipLq0ro/K1BSXYoylXE9czuRGJ6OHvB0dEeIpz+8ZFHwcvKAp8wdno7u8Krb5iQxvZ45tY6OPlaoY9JoNajW1D5crLqmGtU1ytrXNUpU1SgNXlfXVKNKo4Syrr1YXQxljRJVddtNSRGqJ0CAk8QRrwxfjB6eIRCLmX5H1JC1fa5YLICXyWSoqTFOI6gPqusD+YYEQcDq1asxbtw49O/fv8VrqNXGj16vv05T12gOF7G2vQUTo+AstcPe7y7h+s2yeyozWZtnXm1QmaWx1JYydbnRB50IIrg4OOtTVwK9AxuUTKxNZ3GWODX/q2w1UKXWogoV9/I20D3qTGOFOiYRHOAEBzjBvfZT2h6AGev7a1OEahp9lsCHpz9t9Jiqmmq467xRVFTZKvdA1JFwEetdfHx8Gk1hKSwsBIAm898PHz6MrKwsLF26FHl5eQbbKioqkJeXhy5dukAmk8HHxwc1NTVQKBQGaTRqtRoKhaLFRbJkGS2VmVRp1foqLA3TWe4O2Bt7qI1TXZ65u4MbIj3DGzxkqLadeeZEZMtqU4Qc4GDnYJQi5Cn1QIlKYXSMp9TDqI2IrJfFAvjIyEjs2LEDlZWVBgtZT506pd/emPz8fOh0Ojz++ONG29LT05Geno6tW7fioYceQlRUFADgzJkzGDZsmH6/M2fOQKfT6beTddDoNHfqmavKIAsow4CRxci6dhp/PXIUnl5ApaYc1Rql0bEOYom+Ekt3t+AGTwF115dQ5INEiKgzmxwWj5ScPQYTHBKxBJPD4i3YKyIyl8UC+Pj4eGzbtg27d+/W14FXq9VIT09H37599Qtc8/PzUV1djbCwMADAqFGjEBRkvLjx+eefx8iRI5GYmIjo6GgAwODBg+Hh4YGUlBSDAH7nzp1wcnLCQw891MZ3SUBtnnm5uhKl6lLDVJa62fL6haEVNca/urUT2cHdzxnlpXYouiVDn+69EeLZxXDWXOoGmV3Hq2dORNTaBnbtCwBGVWjq24nINlgsgI+NjUV8fDzWrl2LwsJChISEYO/evcjPz8ebb76p3+/ll19GRkYGzp8/DwAICQlBSEhIo+cMDg7GmDFj9K9lMhkWL16MVatWYcmSJRg2bBgyMzOxf/9+LFu2DG5uHbveeFsTBAFVmuoGqSzGqS1N5Zm7OrjAXeoGL5kHQt1CGqnMcifP/EZRJdannkJmrhr9pkQjrrttlpkkIrK0gV37YmDXvlwvQmTDLPoEhjVr1uDdd9/Fvn37UFpaioiICGzZsgX9+vVrtWvMmTMHEokE27Ztw5EjR+Dv74+VK1ciKSmp1a7REam06jv1yxuraV7X1ljZM2d7J30w7u/kd1cqy52ngLpKXMzKM7f2MpNERERE7UUkCEL7l1SxYbZehaZGp0GZqtwgdeVOKsud1BaltpE8czsH/ULPu4Px+te1Dxpq2zxzlVqLD/efxX8v3Ub8wBAkjgyDmKkzdBfOKhKZhmOFyDSsQkNma+yJeY3lKtbmmVcY5ZU3rNLSZJ55XWAe4OyHKK9wo1SW2jxzqcXzzKUOdlg0rTdSvr2AQxnXcLtMeU9lJomIiIhsFQN4K5Zx86RBtYASlQL/zt6NrMKzcHFwMZg9byrP3O3uPHP3bnfVMr8ze+4scbJ4YG6OlspMEhEREXVkDOCt2P7cQ0a1zLWCFr8VnoazxEmfuuLv4qd/6NDdlVnMzTO3JSKRCPGDQuDlJsW/vsjG6h2/YumjsfD1MONpJ0REREQ2iAG8FWvsYRv11jz4evt1xIoNjPKDh4sUG/dkYfX2TCxJjEWPAFYXIiIioo6rmefAk6U19WQ8PjHPkDzYA3+d2w9SiR3WpJzEbxcLLd0lIiIiojbDAN6KTQ6Lh0RsmNfNJ+Y1rr7MZKCPMzaln8aRX/Ms3SUiIiKiNsEA3ooN7NoXsyOnw1PqARFqZ95nR07nE/Oa4O7sgJce64vYsC5IPnwBqUcvQccqqURERNTBMAfeyvGJeeZhmUkiIiLq6BjAU4fDMpNERETUkTGFhjqk+jKTz0yJxtUb5Vi941cUKKot3S0iIiKi+8YAnjq0gVF+WDarDyqq1Fi9PROX88ss3SUiIiKi+8IAnjo8lpkkIiKijoQBPHUKLDNJREREHQUDeOo0WGaSiIiIOgIG8NSp1JeZHNU3EIcyruGDfWdRo9FaultEREREJmMZSep0WGaSiIiIbBln4KlTYplJIiIislUM4KlTY5lJIiIisjUM4KnTY5lJIiIisiUM4InAMpNERERkOxjAE9VhmUkiIiKyBQzgie7CMpNERERk7VhGkqgBlpkkIiIia8YZeKJG1JeZfHZqL5aZJCIiIqvCAJ6oGQMifVlmkoiIiKwKA3iiFrDMJBEREVkTBvBEJmCZSSIiIrIWDOCJTMQyk0RERGQNGMATmYFlJomIiMjSWEaSyEwsM0lERESWxBl4onvAMpNERERkKRadgVer1Xjvvfewb98+lJWVITIyEkuXLsWQIUOaPW7//v1IS0tDbm4uSktL4evri0GDBmHRokUIDAw02DciIqLRc7z++ut47LHHWu1eqHMaEOkLd2cHbNyThdXbM7EkMRY9Atws3S0iIiLqwCwawC9fvhzffPMNkpKS0K1bN+zduxcLFizAjh07EBcX1+RxOTk58PPzw/Dhw+Hu7o78/Hykpqbiu+++w/79++Hj42Ow/7BhwzB58mSDttjY2Da5J+p86stMrk89hTUpJ7FwSjTiwn1aPpCIiIjoHogEwTJlNLKysjBjxgysWLEC8+bNAwCoVCokJCTA19cXycnJZp3v7NmzmDZtGl566SXMnz9f3x4REYGkpCSsXLmyVfpdVFQBna793zIfH1cUFpa3+3XJdKWVamxIO4WrN8sxe4wco/sFWbpLnRLHCpFpOFaITGOJsSIWi+Dt7dL09nbsi4FDhw5BIpFgxowZ+japVIrExET8+uuvKCgoMOt8AQEBAICyssaflKlUKqFSqe69w0QtYJlJIiIiag8WC+Czs7MRGhoKZ2dng/aYmBgIgoDs7OwWz6FQKFBUVITTp09jxYoVANBo/nxaWhr69OmDmJgYTJo0CYcPH26dmyBqgGUmiYiIqK1ZLAe+sLAQfn5+Ru31+eumzMCPHz8eCoUCAODh4YHXXnsNgwcPNtgnLi4OEyZMQFBQEG7cuIHt27dj0aJFWLduHRISElrhTogMscwkERERtSWLBfBKpRISiXFAI5VKAcCkdJdNmzahqqoKV65cwf79+1FZWWm0z65duwxeP/LII0hISMDbb7+NiRMnQiQSmdXv5vKR2pqPj6vFrk3mm5sQjdBgD7yTchJvpZzE6wuGoKu3c8sH0n3jWCEyDccKkWmsbaxYLICXyWSoqakxaq8P3OsD+eYMGDAAADB8+HCMHj0akyZNgpOTE/785z83eYyTkxNmzZqFdevW4fLlywgLCzOr31zESuaICHDDizP7YOOeLLzw7nGWmWwHHCtEpuFYITINF7HexcfHp9E0mcLCQgCAr6+vWecLDg5GdHQ0Dhw40OK+/v7+AIDS0lKzrkF0L+rLTEoldliTchK/XSy0dJeIiIjIhlksgI+MjMSVK1eM0l5OnTql324upVKJ8vKWvyFdv34dAODl5WX2NYjuhb+3M1Ym9UegjzM2pZ/GkV/zLN0lIiIislEWC+Dj4+NRU1OD3bt369vUajXS09PRt29f/QLX/Px85ObmGhxbXFxsdL4zZ84gJycH0dHRze5XUlKClJQUBAUFoXv37q10N0QtY5lJIiIiag0Wy4GPjY1FfHw81q5di8LCQoSEhGDv3r3Iz8/Hm2++qd/v5ZdfRkZGBs6fP69vGzlyJB5++GHI5XI4OTnh0qVL2LNnD5ydnfHcc8/p90tOTsaRI0cwYsQIBAQE4NatW/jss89QXFyM999/v13vlwi4U2Yy5dsLOJRxDbfLlFiQEAWJvZ2lu0ZEREQ2wmIBPACsWbMG7777Lvbt24fS0lJERERgy5Yt6NevX7PHzZ49Gz///DO+/fZbKJVK+Pj4ID4+Hs899xyCg4P1+8XFxeHkyZPYvXs3SktL4eTkhD59+mDhwoUtXoOorbDMJBEREd0PkSDwd/jmYBUaak3/ySnA1gPn4O0uw9JHY+Hr4WjpLtk8jhUi03CsEJmGVWiIyMCASF8sm9UHFVVqrN6eicv5ZZbuEhEREVk5BvBEFsYyk0RERGQOBvBEVoBlJomIiMhUDOCJrATLTBIREZEpGMATWZH6MpOj+gbiUMY1fLDvLGo0Wkt3i4iIiKyIRctIEpExlpkkIiKi5nAGnsgKiUQixA8KwbNTe+HqjXKs3vErChTVlu4WERERWQEG8ERWjGUmiYiIqCEG8ERWjmUmiYiI6G4M4IlsAMtMEhERUT0G8EQ2gmUmiYiICGAAT2RTWGaSiIiIWEaSyMawzCQREVHnxhl4IhvEMpNERESdFwN4IhvGMpNERESdDwN4IhvHMpNERESdCwN4og6AZSaJiIg6DwbwRB0Ey0wSERF1DgzgiToQlpkkIiLq+FhGkqiDYZlJIiKijo0z8EQdEMtMEhERdVwM4Ik6MJaZJCIi6ngYwBN1cCwzSURE1LEwgCfqBPy9nfEKy0wSERF1CAzgiToJN5aZJCIi6hBaJYDXaDT4+uuvkZqaisJC/nqeyFqxzCQREZHtM7uM5Jo1a3DixAns2bMHACAIAp544glkZmZCEAR4eHggNTUVISEhrd5ZIrp/LDNJRERk28yegf/hhx/Qv39//eujR4/iP//5D+bPn49169YBALZs2dJ6PSSiVscyk0RERLbL7Bn4mzdvolu3bvrXx44dQ1BQEJYtWwYAuHjxIg4cONB6PSSiNjMg0hfuzg7YuCcLq7dnYkliLHoEuFm6W0RERNQMs2fga2pqYG9/J+4/ceIEHnjgAf3r4OBg5sET2RCWmSQiIrItZgfwXbt2xW+//Qagdrb9+vXrGDBggH57UVERnJycWq+HRNTmWGaSiIjIdpidQjNx4kRs3rwZxcXFuHjxIlxcXDB8+HD99uzsbJMXsKrVarz33nvYt28fysrKEBkZiaVLl2LIkCHNHrd//36kpaUhNzcXpaWl8PX1xaBBg7Bo0SIEBgYa7b97925s27YNeXl5CAgIQFJSEubMmWPejRN1cPVlJj/cfxbJhy+gqFSJxJFhEItElu4aERER3cXsGfiFCxfikUcewX//+1+IRCK89dZbcHOrzZktLy/H0aNHWwzA6y1fvhyffvopJk+ejJUrV0IsFmPBggX6Gf6m5OTkwM/PD08++SRef/11TJ06FT/88AMSExON0nd27dqFV155BXK5HK+++ipiY2OxatUqbNu2zdxbJ+rwWGaSiIjI+okEofWe5KLT6VBZWQmZTAaJpPmSdFlZWZgxYwZWrFiBefPmAQBUKhUSEhLg6+uL5ORks6599uxZTJs2DS+99BLmz58PAFAqlRg+fDj69euHzZs36/ddtmwZjh49iuPHj8PV1dWs6xQVVUCna/+H3/j4uKKwsLzdr0udkyAI+DrjOlKPXULPIHebKjPJsUJkGo4VItNYYqyIxSJ4e7s0vb01L6bRaODq6tpi8A4Ahw4dgkQiwYwZM/RtUqkUiYmJ+PXXX1FQUGDWtQMCAgAAZWVl+rYTJ05AoVBg9uzZBvvOmTMHlZWV+P777826BlFnwTKTRERE1svsAP748ePYuHGjQVtycjL69u2LPn364MUXX0RNTU2L58nOzkZoaCicnZ0N2mNiYiAIArKzs1s8h0KhQFFREU6fPo0VK1YAgEH6zrlz5wAAvXr1MjguOjoaYrFYv52IGjcg0hfLZvVBRZUaq7dn4nJ+WcsHERERUZsyO4D/6KOPcPnyZf3r3Nxc/OMf/4Cvry8eeOABHDx40KT0l8LCQvj6+hq1+/j4AIBJM/Djx4/HAw88gMTERPz222947bXXMHjwYINrODg4wMPDw+C4+jZzZ/mJOiOWmSQiIrIuZlehuXz5skHVmYMHD0IqlSItLQ0uLi548cUX8fnnn+vz2puiVCobTbWRSqUAavPhW7Jp0yZUVVXhypUr2L9/PyorK026Rv11TLlGQ83lI7U1Hx/z8vWJWouPjyvWL/XEqo9+wfvpp/H01N6YOKyHpbvVJI4VItNwrBCZxtrGitkBfGlpKTw9PfWvf/rpJwwePBguLrWB7cCBA3H8+PEWzyOTyRpNtakPqusD+ebU158fPnw4Ro8ejUmTJsHJyQl//vOf9ddQq9WNHqtSqUy6RkNcxEqd2QszYvHh/rP4YO9pXP2j1CrLTHKsEJmGY4XINB1iEaunpyfy8/MBABUVFTh9+jT69++v367RaKDVtlx2zsfHp9EUlvoykI2l1zQnODgY0dHROHDggME1ampqoFAoDPZVq9VQKBRmX4Oos2OZSSIiIsszO4Dv06cPdu3ahUOHDuEf//gHtFotHnroIf3233//3aTAODIyEleuXDFKezl16pR+u7mUSiXKy+98Q4qKigIAnDlzxmC/M2fOQKfT6bcTkenEYhHmjJXj0ZE9kZlTgLd3/RcV1S0vXCciIqLWYXYAv3jxYuh0Ovy///f/kJ6ejqlTp6Jnz54AamtHf/vtt+jbt2+L54mPj0dNTQ12796tb1Or1UhPT0ffvn3h5+cHAMjPz0dubq7BscXFxUbnO3PmDHJychAdHa1vGzx4MDw8PJCSkmKw786dO+Hk5GTwxYOITMcyk0RERJZjdg58z549cfDgQZw8eRKurq76PHSgtgb7448/jkGDBrV4ntjYWMTHx2Pt2rUoLCxESEgI9u7di/z8fLz55pv6/V5++WVkZGTg/Pnz+raRI0fi4Ycfhlwuh5OTEy5duoQ9e/bA2dkZzz33nH4/mUyGxYsXY9WqVViyZAmGDRuGzMxM7N+/H8uWLdM/QZaI7s2ASF+4Oztg454srN6eiSWJsegRwHFFRETUllr1SazmUqlUePfdd3HgwAGUlpYiIiICL7zwAh544AH9PnPnzjUK4N966y38/PPPyMvLg1KphI+PDwYPHoznnnsOwcHBRtdJTU3Ftm3bkJeXB39/f8ydOxdJSUn31GcuYiUydqOoEutTT6GsUo2FU6IRF+5jsb5wrBCZhmOFyDTWuIj1ngP4a9eu4ciRI7h+/TqA2kWko0ePRkhIyL311EYwgCdqXFmlGu+lncLVm+WYPUaO0f2CLNIPjhUi03CsEJmmwwTw7777LrZu3WpUbUYsFmPhwoVYsmSJ+T21EQzgiZqmUmvx4f6z+O+l24gfGGKRMpMcK0Sm4VghMo01BvBm58CnpaXhgw8+QFxcHJ566imEh4cDAC5evIiPPvoIH3zwAYKDgzFt2rR77zUR2aT6MpMp317AoYxruF2mxIKEKEjs7SzdNSIiog7D7Bn4adOmQSKRIDk5Gfb2hvG/RqPBnDlzUFNTg/T09FbtqLXgDDxRywRBwNcZ15F67BJ6Brlj8fQYuDg2/lTk1saxQmQajhUi01jjDLzZZSRzc3MxYcIEo+AdAOzt7TFhwgSjso9E1LmwzCQREVHbMTuAl0gkqKqqanJ7ZWUlJJL2mWkjIus2INIXy2b1QUWVGqu3Z+Jyfpmlu0RERGTzzA7ge/fujc8++wy3b9822lZUVITU1FTExsa2SueIyPbJgz3w17n9IJXYYU3KSfx2sdDSXSIiIrJpZufA/+c//8G8efPg7OyM6dOn65/CeunSJaSnp6OyshKffPIJ+vfv3yYdtjTmwBPdm/YqM8mxQmQajhUi01hjDvw9lZE8evQo/v73v+PGjRsG7QEBAXjttdcwYsQIsztqKxjAE9279igzybFCZBqOFSLTWGMAb3YZSQAYNWoURowYgTNnziAvLw9A7YOcoqOjkZqaigkTJuDgwYP31mMi6rBYZpKIiOj+3VMAD9Q+tCkmJgYxMTEG7SUlJbhy5cp9d4yIOiaxWIQ5Y+Xo4u6I1GOXoKhQtWuZSSIiIltn9iJWIqL7xTKTRERE944BPBFZDMtMEhERmY8BPBFZFMtMEhERmYcBPBFZnL+3M15J6o9AH2dsSj+NI7/mWbpLREREVsukRawff/yxySc8efLkPXeGiDovN2cHvPRYX3y4/yySD19AUamyTcpMEhER2TqTAvi33nrLrJOK+IFLRPeAZSaJiIhaZlIAv3379rbuBxERAJaZJCIiaolJAfzAgQPbuh9ERHr1ZSa93WXYeuAcVu/4FUsfjYWvh6Olu0ZERGRxXMRKRFaLZSaJiIiMMYAnIqvGMpNERESGGMATkdVjmUkiIqI7GMATkU2oLzMZG9YFyYcvIPXoJegEwdLdIiIiancM4InIZtSXmRzVNxCHMq7hg31nUaPRWrpbRERE7cqkKjRERNaCZSaJiKizYwBPRDansTKTI+IC8O1/rqO4TAUvNymmDQ/DkOiulu4qERFRq2MAT0Q2a0CkL9ydHfDOZ7/hsyOX9O1FZSp8+lUOADCIJyKiDoc58ERk0+TBHnCSGafPqDU6pB/PtUCPiIiI2hYDeCKyeYoKdaPtRWUqaHW6du4NERFR22IAT0Q2z9tN2uS2Ze//hN3fXcKNosp27BEREVHbYQ48Edm8acPD8OlXOVBr7sy2O9iLMaJPAAoUSnx94jq++uUaega548HeyhyR1QAAIABJREFU/ugf6QtHKf/5IyIi28RPMCKyefULVdOP5zZahaa0QoWfzt7Ej1k38PFXOUj59iIGRPpiWIw/woPcIRKJLNl9IiIis4gEwXKPMlSr1Xjvvfewb98+lJWVITIyEkuXLsWQIUOaPe6bb77BwYMHkZWVhaKiIvj7+2PkyJF47rnn4OrqarBvREREo+d4/fXX8dhjj5nd56KiCuh07f+W+fi4orCwvN2vS2RrmhsrgiAgN78MP2bl40R2AVRqLfy8nPBgjD8e6NUVHi5Np+IQdTT8XCEyjSXGilgsgre3S5PbLRrAv/DCC/jmm2+QlJSEbt26Ye/evThz5gx27NiBuLi4Jo8bNGgQfH19MWbMGAQEBOD8+fPYtWsXunfvjj179kAqvfMhHBERgWHDhmHy5MkG54iNjUX37t3N7jMDeCLrZupYUam1yDxfgB+ybuDCdQXEIhF69/DCsJgAxPb0hr0dlwhRx8bPFSLTWGMAb7EUmqysLHz55ZdYsWIF5s2bBwCYOnUqEhISsHbtWiQnJzd57IYNGzBo0CCDtl69euHll1/Gl19+iWnTphls69GjB6ZMmdLq90BEtkvqYIehvf0xtLc/bhVX4cfTN/B/p2/g1N7TcHWSYEh0VzwY449An6b/ASUiIrIEiwXwhw4dgkQiwYwZM/RtUqkUiYmJWL9+PQoKCuDr69vosQ2DdwAYM2YMACA3t/G6z0qlEiKRyGB2nogIAPy8nDB9eBimPhiKs1eK8UPWDRz5NQ/f/Oc6Qv3d8GCsPwZG+sFJxmVDRERkeRb7NMrOzkZoaCicnZ0N2mNiYiAIArKzs5sM4Btz+/ZtAICnp6fRtrS0NOzYsQOCIEAul2Px4sUYO3bs/d0AEXU4dmIxYsK6ICasC8qq1Pjl7C38kJWP7YfOY9e3F9EvwhcPxvhDHuIBMRe+EhGRhVgsgC8sLISfn59Ru4+PDwCgoKDArPNt3boVdnZ2GDdunEF7XFwcJkyYgKCgINy4cQPbt2/HokWLsG7dOiQkJNz7DRBRh+bm5IBxA4Ixtn8Qrt4sx49ZN/DLuVv4+exN+HjIMKwu/cbLTWbprhIRUSdjsQBeqVRCIjF+/Hl9iotKpTL5XAcOHEBaWhoWLlyIkJAQg227du0yeP3II48gISEBb7/9NiZOnGh2+bjmFhS0NR8f15Z3IqJWHyu+vm4YGBMIVY0WP5++gW8zfsfeH67g8x+vIE7uizEDQzC4V1dI7O1a9bpEbY2fK0SmsbaxYrEAXiaToaamxqi9PnA3NVc9MzMTK1euxIgRI7BkyZIW93dycsKsWbOwbt06XL58GWFhYWb1m1VoiKxbW4+V6GB3RAfHoFBRjf87fQM/nr6BNTsy4Syzx+C6ha8hftb1Dz1RY/i5QmQaVqG5i4+PT6NpMoWFhQBgUv57Tk4Onn32WURERGD9+vWwszNt9svf3x//f3v3HhdlnfYP/DPnGWA4DzCIgKCCIiCeETDNQ+QhrbW1Mt3NtCy1g9s+bvU8z7YH137lpmbpmtZTmWV5QNTMsyVoaWIxIgKJihDDWc6ngZnfH8jgBCooeM/A5/167WtffO/7nvs7xBcvrrm+1w0AZWVlHZgxEVELjbMKM2IC8FBUH1zIuoYEXS6++zkXR5Jy4OvpgJgwb4wc6AkHVetPGomIiO6GYAF8cHAwNm/ejKqqKouNrMnJyebjt3L16lXMnz8frq6u2LBhA+zs7Np97+zsbACAq6vrHcyciKiFWCxCSB9XhPRxRWWNAadS85Go02PLoQx8efQihvR3R0yYNwb4u3DjKxERdQrBnlQSGxsLg8GAbdu2mcfq6+uxc+dODBkyxLzBNTc3t1VryMLCQsybNw8ikQgffvjhTQPxkpKSVmPXrl3D559/Dh8fnzt6kBMR0c04qGQYP9QHf31qON54ajjuG+yN85dL8O8vf8ay9SexK+ESCktrhJ4mERHZOMEy8OHh4YiNjcXKlStRWFgIX19fxMXFITc3FytWrDCft2zZMpw+fRrp6enmsfnz5yM7Oxvz589HUlISkpKSzMd8fX3NT3HdsmULjhw5grFjx8Lb2xv5+fn48ssvUVJSgvfff//evVki6nF8PdWYPVGN348LxE+/FCFRp8eeE1ew+8QVDPBzQXSYFkP7ayCXceMrERF1jKBPJXnrrbewevVqxMfHo6ysDEFBQfjggw8wdOjQW16XlpYGANi0aVOrYw8//LA5gI+IiMDZs2exbds2lJWVwc7ODoMHD8azzz5723sQEXUGmVSCEQM8MWKAJ0rKa3HinB4JOj027knFZwopRg30RHSYFv5e6g53xSIiop5JZDKZ7n1LFRvGLjRE1s0W1orRZELG1VIk6PRISi9AfYMRPhp7RId5Y1SIJxzt5EJPkXoAW1grRNbAGrvQMIDvIAbwRNbN1tZKdW0DTl/IR4JOj8v6ckjEIgzu546YMC1C+rhCIhZsqxJ1c7a2VoiEYo0BvKAlNEREPZ2dUoqxEb0wNqIXcgorkajT4/vzeUhKL4SzgxxRoVpEh2nh6dL+TltERNS9MQPfQczAE1m37rBWGhqNSL5YjERdLnSXimEyAf17OyMmTIthQR5QyLnxle5ed1grRPeCNWbgGcB3EAN4IuvW3dbKtYo6nEzRI1GnR/61GijkEowc4IHoMG8Eejty4yvdse62Voi6ijUG8CyhISKyYi5qBaZE+mPyKD/8klOGRJ0ep1ILcDxZD62bHaLDtBgd4gUnB4XQUyUionuEGfgOYgaeyLr1hLVSU9eAM2kFSDinx8WcMohFIoQFuiEmTIvQQDdIJdz4SrfXE9YKUWdgBp6IiO6aSiFFTLg3YsK9oS+uQuI5PU6ey8PPF4vgaCfD6EFNG1+93e2FnioREXUBZuA7iBl4IuvWU9dKo9GIc5dKkKjTI/liERqNJgT2ckRMmDeGB3tApWC+hiz11LVC1FHMwBMRUZeQiMUY3Ncdg/u6o7yqHidT8pB4To+Pv0nD54czMDzIA9FhWvTv7cyNr0RENo4BPBFRN+NoL0fsSF88MKI3LunLr298zceJlDx4uKgQHapFVKgWLmpufCUiskUsoekgltAQWTeulbbVGRqRlF6ARJ0eaVdLIRIBoQFuiA7VYnA/d2587YG4VojahyU0REQkCIVMgtGDtBg9SIuCa9VIPJeHE+f0WLcrBQ4qGSJDvBATpoWPx83/wSAiIuvADHwHMQNPZN24VtrPaDTh/JUSJOj0+CmjEI1GE/y91IgJ02LkQE/YKWVCT5G6ENcKUfswA09ERFZDLBYhNMANoQFuqKiuxw+p+UhI1mPzwQxsPXoRQ/trEBOmRZCfC8Tc+EpEZDUYwBMREdR2ckwc1hsThvrgan4lEnS5+OF8Pn5IzYe7kxLRoVqMDvWCu5NK6KkSEfV4LKHpIJbQEFk3rpXOY2hoxNmMIiTocnHhyjUAwEB/F0SHeWNIf3fIpBKBZ0h3g2uFqH1YQkNERDZDJpVg5EBPjBzoiaLSGpxIyUOiTo8Nu8/DTiHFqBBPxIR5w89LLfRUiYh6FAbwRER0W+7OKkyP7oNpUf5Iy7qGRJ0ex5P1OHr2V/h6OCA6TItRIV5wUHHjKxFRV2MJTQexhIbIunGt3DtVtQacTs1Hgk6PK3kVkEpEiOjXtPF1oL8rxGJufLVmXCtE7cMSGiIi6jbslTKMG+KDcUN8kF1QiUSdHt+fz8OPaQVwUSsQFapFdKgXPFzshJ4qEVG3wgx8BzEDT2TduFaEZWgwIvliERJ0eqRcLobJBAT7OiM6TIuhQR5QyLjx1VpwrRC1jzVm4BnAdxADeCLrxrViPUrKa3Hy+sbXgtIaqBQSjBjgiegwLQK0jhCxt7yguFaI2scaA3iW0BARUZdwdVRi6mh/TIn0Q0Z2qbnE5rufc9HL3R7RYVpEhnjB0V4u9FSJiGwKM/AdxAw8kXXjWrFuNXUN+DGtAAm6XGT+Wg6JWITwvu6IDtMiNMAVErFY6Cn2GFwrRO3DDDwREfVoKoUUY8K9MSbcG78WVeGETo+TKXqczSiEk70co0O9EB2qhdbNXuipEhFZLWbgO4gZeCLrxrViexoajTiXWYwEnR66zGIYTSb09XFCTKgWwwd4QClnrqkrcK0QtY81ZuAZwHcQA3gi68a1YtvKKutw8nweEpL1yCuphkImwfABHogJ06JvLydufO1EXCtE7WONATzTGkREZDWcHBR4cKQfYkf4IjO3HAnJuTidVoBEnR6ernaICdNi9CAvODsohJ4qEZFgmIHvIGbgiawb10r3U1vfgDNphUjU5SIjpwxikQihAa6IDvNGeF83SCXc+HonuFaI2ocZeCIiog5SyqWIDtMiOkyLvJJqnDinR+I5PZLjzkFtJ8PoQU0bX3tpbv6PHRFRd8IMfAcxA09k3bhWeoZGoxHnL5cgQafHz78UodFoQoC3I6LDtBgR7Ak7JfNTt8O1QtQ+zMD/Rn19PdasWYP4+HiUl5cjODgYL7/8MiIjI2953cGDB7Fv3z7odDoUFxdDq9Vi3LhxeP7556FWq1udv23bNnz00UfIycmBt7c35s6di9mzZ3fV2yIioi4mEYsRFuiOsEB3lFfX44eUPCTo9Ph0fzq2Hv4FQ4OaNr7293WGmBtfiaibETQDv3TpUhw8eBBz586Fn58f4uLikJKSgs2bNyMiIuKm140cORIeHh6YMGECvL29kZ6ejq1bt8Lf3x87duyAQtGyuWnr1q3461//itjYWERFReHMmTOIj4/HsmXLMG/evA7PmRl4IuvGtdJzmUwmXMmrQIJOj1Opeaipa4TGWYnoUC2iQrVwdVQKPUWrwrVC1D7WmIEXLIDX6XR49NFH8eqrr+KPf/wjAKCurg5Tp06Fh4cHtmzZctNrT506hZEjR1qM7dq1C8uWLcOKFSvwyCOPAABqa2tx3333YejQoVi3bp353FdeeQVHjx7Fd99912bG/lYYwBNZN64VAoA6QyPOZhQiUafHhaxrEAEICXBFTJg3Bvd1h0zKja9cK0TtY40BvGC/wfbv3w+ZTIZHH33UPKZQKDBz5kwkJSWhoKDgptf+NngHgAkTJgAAMjMzzWOnTp1CaWkpnnjiCYtzZ8+ejaqqKhw/fvxu3wYREVkhhUyCyBAv/PnxCPy/hZGYFuWP3KIqrN+VgqXvJeLzQxm4ms/glYhsk2A18BcuXECfPn1gb2/5uOywsDCYTCZcuHABHh4e7X69oqIiAICLi4t5LDU1FQAwaNAgi3NDQkIgFouRmpqKKVOm3OlbICIiG6BxVmFGTAAeiuqDC1nXkKDLxbc//4rDSTnw81QjOkyLUSGesFfKhJ4qEVG7CBbAFxYWwtPTs9W4RqMBgFtm4NuyceNGSCQSTJo0yeIecrkczs7OFuc2j3X0HkREZLvEYhFC+rgipI8rKmsMOJWajwRdLrYcysCXRy9iSH93xIR5Y4C/Cze+EpFVEyyAr62thUzWOtvRvAG1rq6u3a+1Z88ebN++Hc8++yx8fX1ve4/m+3TkHs1uVY/U1TSajtXrE/VUXCt0OxoAfXxd8VjsAGTmlOLwj1fxbVIOTl8ogMZFhfHDfDFhhC88Xe2EnmqX4lohah9rWyuCBfBKpRIGg6HVeHNQfWMnmVs5c+YMXn/9dYwdOxYvvvhiq3vU19e3eV1dXV2773EjbmIlsm5cK9RRjgoJHonug2mjfPHTL0VI1Onx5aF0bD2UjgF+LogJ02JIfw3kMonQU+1UXCtE7WONm1gFC+A1Gk2bJSyFhYUA0K7697S0NDz33HMICgrCqlWrIJFY/nLVaDQwGAwoLS21KKOpr69HaWlph2rsiYioe5NJJRgxwBMjBniiuKwWJ1P0SNDp8cGeVKgUUowa6InoMC38vdQQscSGiAQkWAAfHByMzZs3o6qqymIja3Jysvn4rVy9ehXz58+Hq6srNmzYADu71h9zDhgwAACQkpKC6Oho83hKSgqMRqP5OBER0Y3cnJSYFtUHU0b7I/1qKRJ1uUg8p8exn36Fj8Ye0WHeiAzxhNpOLvRUiagHEqyNZGxsLAwGA7Zt22Yeq6+vx86dOzFkyBDzBtfc3FyL1pBAU5Z+3rx5EIlE+PDDD+Hq6trmPUaNGgVnZ2d8/vnnFuNffPEF7OzsMGbMmE5+V0RE1J2IRSIM8HPBgmkhWLU4GnMfCIJMKsHWI79g6XsnsC7uHHSZxYKUVhJRzyVYBj48PByxsbFYuXIlCgsL4evri7i4OOTm5mLFihXm85YtW4bTp08jPT3dPDZ//nxkZ2dj/vz5SEpKQlJSkvmYr6+v+SmuSqUSL7zwAv7+97/jxRdfRHR0NM6cOYPdu3fjlVdegaOj4717w0REZNPslFKMjeiFsRG9kFNYiUSdHidT8nAmvRAuagVGD/JCdJgWni7de+MrEQlPsCexAk0bSVevXo09e/agrKwMQUFBWLp0KUaPHm0+Z86cOa0C+KCgoJu+5sMPP4w333zTYuyrr77CRx99hJycHGi1WsyZMwdz5869ozlzEyuRdeNaoXupodGI5ItFSNDpce5SMUwmoH9vZ8SEaTEsyAMKufVufOVaIWofa9zEKmgAb4sYwBNZN64VEsq1ijqcTNEjUadH/rUaKOUSjBjggegwbwR6O1rdxleuFaL2scYAXrASGiIiou7ERa3AlEh/TB7lh19yypCo0+NUagGOJ+uhdbNDTJg3Igd5wcmeG1+J6O4wA99BzMATWTeuFbImNXUNOJNWgASdHhd/LYNYJEJ4XzdEh2kRGuAGqUSwXhJcK0TtxAw8ERFRD6JSSBET7o2YcG/oi6vMG19/+qUIjvZyjB7khZgwLbRu9rd/MSKi65iB7yBm4ImsG9cKWbtGoxHnMkuQoMuFLrMYjUYTAns5IibMG8ODPaBS3JvcGtcKUftYYwaeAXwHtSeAr6mpQmVlKRobGzrtvmKxGEajsdNej4QlkUjh4OAMlYpZt87GoIRsSVlVPb5PyUOCLhf64mrIZWIMD/JATLg3+vk4denGV64VovaxxgCeJTSdrKamChUV1+DsrIFMJu+0X75SqRgNDQzguwOTyQSDoR6lpYUAwCCeqAdzspcjdqQvHhjRG5f05dc3vubjREoePFxUiAnTYvQgLVzUCqGnSkRWhBn4DrpdBr6w8Fc4OblDLu/cX7YM4Luf+vo6lJUVQaPpJfRUuhVmFcnW1dU3IimjAAnJeqRnl0IkAkID3BAdqsXgfu6dtvGVa4WofZiB7wEaGxsgk7FFGN2eTCbv1DIrIuoeFHIJRg9qyrznX6vGiXN6nDiXh3W7UuCgkiEypGnjq4/Hzf9xJ6LujQF8F7C2h3WQdeLPCRHdjqeLHR4ZE4gZ0QE4f6UECTo9jp7NwaEz2fD3UiMm3BsjB3jATikTeqpEdA8xgCciIrJyYrEIoQFuCA1wQ0V1PX5IzUdCsh6bD6Rj65FfMDRIg5hQLYL8XCBmcoCo22MAT1Zh8eJnAADvvffBPb2WiMjWqO3kmDisNyYM9UFWfgUSdHqcOp+PH87nw91JiehQLaJCtXBzUgo9VSLqIgzg6Zaio4e167xt23ZDq/Xu4tkQEVEzkUgEfy9H+Hs5Yta4vjj7SyESdXrsSryM+MTLGOjvgugwbwzp7w6ZVCL0dImoE7ELTQfdrgtNXl4WvLz8Ov2+QnWhOXBgn8XXX331BfLz9ViyZKnF+Jgx46BSqe74PgaDAQAgk3W8jvNurhVaV/289GTsrEE9XVFpDU6k5CFRp0dxeS3slVKMGuiF6DAt/LzU+P58HnZ+l4mS8jq4OirwyH2BiAzxEnraRFaLXWjI5jzwwGSLr7/99gjKykpbjf9WbW0tlMr2f3x7N8G3LQbuRERdxd1ZhenRfTAtyh9pWdeQqNPju+RcHDmbA1e1AmVV9Wi8nogqLq/DJ9+kAQCDeCIbwgCe7trixc+gsrIS//Vfr2Ht2lVIT0/D7Nlz8fTTzyIh4Vvs3h2HjIx0lJeXQaPxwOTJ0zBnzlOQSCQWrwG01LGfPXsGL7ywEMuXv4XLly9h164dKC8vQ2hoOP7859fg49O7U64FgB07vsLWrVtQXFyEwMBALF78MjZuXG/xmkREtkYsEmGgvysG+rtidq0Bp1Lz8cXhX8zBe7P6BiO+OnoRw4M9Oq3HPBF1LQbwNuD783nYefwSistq4WalH3eWll7Df/3Xy5g0KRaxsVPg6dk0v3379kKlssOsWbNhZ6dCUtIZbNr0H1RVVWHRohdv+7qffPIhxGIJnnhiLioqyvHFF5vxt7/9NzZu/KRTro2L245Vq97C4MFDMGvW49Dr9Xj11VegVquh0Xjc+TeEiMiK2CtluH+IDz47mNHm8bKqejz/znfw0TjAz0sNP081/LzU8NHYs36eyAoxgLdy35/PwyffpKH+ev27tX7cWVRUiL/85X8wdep0i/E33vgnFIqWUpoZM2bi7bf/hbi4bViw4DnI5bd+6FVDQwM++ugTSKVNP6qOjk5Ys2YlLl26iICAvnd1rcFgwKZN6xESEorVq9eZz+vbtx+WL3+DATwRdTtujgoUl9e1GndQyRAdpkVWXgXOpBXgu59zATRl8b3d7eHn5WAO6nt7OEApZ/hAJCSuwHvgxDk9EnX6O7o2M7cMDY2tP+78v30XcPz6L9j2ig5rai3WFZRKJWJjp7QavzF4r66uQn29AeHhEYiP34msrCvo16//LV93ypSHzIE1AISHDwYA5Ob+etsA/nbXpqWloqysDM8//7DFeRMnxuLdd9+55WsTEdmiR+4LtEgKAYBcKsbjE/qZk0ImkwnFZbXIyq9o+l9eJc5lFuPEuTwAgAiAl5sd/DzV8L0e1Pt5OvBhUkT3EAN4K/fb4P1240LRaDwsguBmly5lYuPG9Th79kdUVVVZHKuqqrzt6zaX4jRTqx0BABUVt98Nfrtr8/Ka/qj6bU28VCqFVts1f+gQEQmpOUi/VRcakUgEd2cV3J1VGBrU9EmkyWRCaWU9svIrcDWvKbDPyCnFD6n55us0zkpzlt7PUw1fLzUc7W79KSsR3RkG8PdAVOidZ77/vO5Emx93ujkqsGz2kLudWqe5MdPerKKiAkuWPAM7Owc8/fRC9OrlA7lcjoyMNKxfvxZG4+3bYorFbddetqf76d1cS0TUXUWGeCEyxKtDrfFEIhFc1Aq4qBUY3NfdPF5eXY+r+RXIyqtAVn4lruZV4Ex6ofm4i1phEdT7eanh7CCHiE+LJborDOCt3M0+7nzkvkABZ9U+P/2UhLKyMixf/jYGD275Y0Ov71jpT1fx8mr6oyonJxvh4RHm8YaGBuj1egQG3rpEh4iop3O0k2NQHzcM6uNmHquuNeBqfuUNJTgVSL5YBJP5Ghl8mwP660G9u5OSQT1RBzCAt3LmjzutvAtNW8TipnZkN2a8DQYD4uK2CTUlC8HBA+Hk5ITdu+PwwAOTzSVAhw7tR0VFucCzIyKyTXZKGYL9XBDs52Ieq6tvRHZBpTmgz8qvwP4rV80tLe0U0htKb5o2zHq62kHMoJ6oTQzgbUBkiBdiwr0FeRLr3QgNDYNa7Yjly9/AzJmzIBKJcODAPlhLBYtMJsO8ec9g1aq38dJLz2PcuPHQ6/X45ps96NXLh9kgIqJOopBL0NfHCX19nMxjhoZG5BRWWdTVH07KQUOj0XyNr4eDRQmO1t0OEjF71RMxgKcu4+TkjLfeWoX33luNjRvXQ612xKRJD2LYsBFYunSx0NMDAPzud7NgMpmwdesWvP/+GgQG9sObb76D1atXQi5XCD09IqJuSyaVoI/WEX20juaxhkYj9MXV5ix9Vn4FEnR6HE7KuX6N+IZe9U3/38vdATIpg3rqWUQm7ujrkOLiShiNN/+W5eVlwcvLr9PvK5WKbS4Db6uMRiOmTp2I++4bh2XL/rtL79VVPy89WUc25hH1ZLayVoxGE/Kv3RDUX98wW1PXAACQiEXo5W7fUlfvpUZvjQMUcj6AijqHEGtFLBbBzc3hpseZgacera6uDgqFZaZ9//6vUV5ehoiIoQLNioiImonFImjd7KF1s8eoG3rVF5bVmktvsvIq8PMvReZnrohEgNbNvilLb34AlRp2SoY91D3wJ5l6NJ3uZ6xfvxZjx94PR0cnZGSk4euvdyMgIBDjxk0QenpERNQGkUgED2cVPJxVGBbc0qv+WkWdOaC/ml+JtKul+P58S696DxeVZa96Tweo2auebBADeOrRvL17wd1dg+3bv0R5eRkcHZ0QGzsFCxcuhkzGpwoSEdkKkUgEV0clXB2ViOinMY+XVd3Yq74Cl/Xl+DGtwHzczVFxwxNlm3vVcw8UWTcG8NSj9erlg7feWiX0NIiIqIs42csRGuCG0ICWXvVVtYbr5TeVFiU4phuu8fNSNwX2nmr4eTnAzZG96sl6MIAnIiKiHsVeKcMAf1cM8Hc1j9XUNZh71TfX1qdcKoHxeq8Pe6XUIkvv56mGxkXFXvUkCAbwRERE1OOpFFL07+2M/r2dzWP1hpZe9c0lOIfOZKOhsSmoV8olFll6P081vNzYq566HgN4IiIiojbIZRIEeDsiwNuyV31uUZVFr/rvfv4V9ddbPculYvT2cGhpa+mpRi+NPaQSBvXUeQQN4Ovr67FmzRrEx8ejvLwcwcHBePnllxEZGXnL63Q6HXbu3AmdToeMjAwYDAakp6e3Oi8nJwfjx49v8zU2btyIMWPGdMr7ICIiop5BKhHD17OpPj7m+pjRaIK+pNqireX3KXk4dvZXAE296pseQNWUpfe93qteLmOverozggbwf/nLX3Dw4EHMnTsXfn5+iIuLw4IFC7B582ZERETc9LrvvvsO27ZtQ1BQEHr37o1Lly7d8j4PPfQQoqOjLcaCg4M75T0QERFRzya+/jCpXu72iBzU1KveaDLzPxIUAAAZ3UlEQVShsLTGnKm/mleBpPRCHE9u6lUvFomgdbczZ+mbetU7QKVgcQTdnmA/JTqdDl9//TVeffVV/PGPfwQAzJgxA1OnTsXKlSuxZcuWm177+OOPY8GCBVAqlVi+fPltA/iQkBBMnz69M6dPREREdFNikQieLnbwdLHDiAGeAJp61ZeU11nU1J+/UoKTKXkAABEAD1e7pgdQmXvVq+GgYltjsiRYAL9//37IZDI8+uij5jGFQoGZM2di1apVKCgogIeHR5vXuru7d/h+1dXVkEqlkMv5wAYiIiK690QiEdyclHBzUmJI/5Ze9aWVdTf0qq9E5q/lOH2hpVe9u5PSXHrTnK13smc805MJFsBfuHABffr0gb29vcV4WFgYTCYTLly4cNMAvqPWrFmDFStWQCQSITw8HK+88gqGDx/eKa9NREREdDecHRRwdlAgLLAlQVlZY7BoaZmVV4GkjMIbrpFbtLT081LDRa1gr/oeQrAAvrCwEJ6enq3GNZqmv0gLCgpaHesosViM6OhoTJw4ER4eHsjKysKHH36Ip556Ch9//DGGDRt21/egjtu3bw/+9a+/Ydu23dBqvQEAM2dOQ0TEULz++hsdvvZunT17Bi+8sBDvvvsfDBnCnwkiIhKeg0qGEH9XhNzQq766tgHZBdcfQJVXgav5FdBdKsb1VvVwUMl+06veARpnFYP6bkiwAL62trbNR9UrFE2PL66rq7vre3h7e+PDDz+0GJs8eTKmTJmClStXYuvWrR1+TTc3h1seLygQQyrtmlZRXfW6t/OnP72IpKQf8c03R6BSqdo858UXn0dKSgr27Ttk/m94M2Jx0y8SicTyeyUSiW77Hm92bXscOnQAxcVFeOyx2Rbjkuutve7kNe+WWCyGRqO+p/fsCfg9JWofrhXb49fbBTe25aitb8AVfTkyc8qQmVOKzF/LcPDHq+Ze9fZKKQJ6OSPQxwmBvZwQ6OMMb40DJGIG9R1hbWtFsABeqVTCYDC0Gm8O3G8XBN4pT09PTJkyBV999RVqampuGpDeTHFxJYxG002PG41GNFzvBduZpFJxl7xue0yY8ABOnEjAt98ew8SJsa2OX7tWgjNnfsSkSQ9CIpHddp7N37/Gxpbv1eef74BYfPv32Na17XXw4H788ksGZs583GI8NHQwjhw5AZns9nPvbEajEYWFFff0nt2dRqPm95SoHbhWug83Oxnc+rtjRP+mEhxDw/Ve9Tdslv36xGUYmnvVy8Tw9bi+SfZ6a0tvd/aqvxkh1opYLLpl0liwAF6j0bRZJlNY2FTf1Vn1723RarUwGo0oLy/vcADfE8XEjIVKZYfDhw+0GcAfPXoYjY2NmDSp9bH2EnJzsVgs7rI/GImIiO41mVTcVELjpQbCm8YajUboi6ubAvrrQX3iOT3qzjYCaOpv76OxtyjB8dHYQyZlr3prJFgAHxwcjM2bN6OqqspiI2tycrL5eFfJzs6GRCKBk5NTl92jO1EqlYiJuQ/Hjh1GeXk5HB0dLY4fPnwAbm5u6N3bDytXvomkpNPIz8+HUqnEkCHDsGjRi7etV2+rBv7SpUysXv02UlLOwcnJCdOnPwJ3d02raxMSvsXu3XHIyEhHeXkZNBoPTJ48DXPmPAWJpOkXz+LFz+Dnn88CAKKjm+rcvby02L59z01r4I8cOYjPPvsYWVlXYGdnj6ioGDz33Atwdm55zPbixc+gsrIS//u/f8c777yFCxfOQ612xKOPPobZs//QsW80ERFRF5GIxfDROMBH44CoUC2Apl71+SXV1zfLViIrvwI/XijAdz/nXr9GBK2bvfkBVM296pVy9qoXmmD/BWJjY/HRRx9h27Zt5j7w9fX12LlzJ4YMGWLe4Jqbm4uamhoEBgZ2+B4lJSVwdXW1GMvKysLXX3+NYcOGQalU3vX7uBdO553Fnkv7UVJbCheFMx4KjMUIryH3dA4TJ8bi4MFv8O23R/DQQw+bx/Py9EhJ0WHmzMdw4cJ5pKToMGHCA9BoPKDX52LXrh1YsuRZfPbZtg59v4uLi/DCCwthNBrx5JN/gFKpwu7dcW1myvft2wuVyg6zZs2GnZ0KSUlnsGnTf1BVVYVFi14EAPzhD/NQU1OD/Hw9lixZCgBQqexuev/mzbIhIaF47rkXUFCQjx07vsSFC+exceOnFvMoLy/Dn/70AsaNG4/x4yfh2LHDWL9+LQIC+iIyMqrd75mIiOheEouaAnStmz1GDWwaM5lMKCqrNWfps/IroMssxolzLb3qvdzszD3qmzfL2inZq/5eEiyADw8PR2xsLFauXInCwkL4+voiLi4Oubm5WLFihfm8ZcuW4fTp00hPTzeP/frrr4iPjwcAnDt3DgCwbt06AE2Z+/vvvx8A8PbbbyM7OxujRo2Ch4cHrl69at64umzZsnvyPu/W6byz+DxtBwzGpv0C1+pK8XnaDgC4p0H88OEj4ezsgsOHD1gE8IcPH4DJZMLEiQ8gMLAvxo2bYHFdVNQYLFz4FL799ghiY6e0+35btnyCsrJSbNq0GUFBTZ/GPPjgVDz++MOtzn3jjX9CoWj542DGjJl4++1/IS5uGxYseA5yuRzDh4/Czp3bUFZWigcemHzLezc0NGD9+rXo27c/1q7dYC7vCQoKxhtvvI49e+Iwc+Zj5vMLCvLx17/+01xeNHXqdMycORVffx3PAJ6IiGyKSCSCxlkFjbMKw4KbyplNJhNKK+tbgvq8CqRnl+KH1HzzdRpnZUv3G6+m4N7Rjr3qu4qgn4G89dZbWL16NeLj41FWVoagoCB88MEHGDp06C2vy8nJwZo1ayzGmr9++OGHzQF8VFQUtm7dis8++wwVFRVwdHREVFQUFi9ejH79+nXNm2rDKX0Svtf/eEfXXi67igZTg8WYwWjAlgvbcTL3dIdeK1I7HCO1t/7e3oxUKsX990/Arl07UFRUZH6Y1uHDB+Hj0xsDBw6yOL+hoQFVVZXw8ekNBwc1MjLSOhTAf//9CYSGhpuDdwBwcXHBxIkPIi5um8W5Nwbv1dVVqK83IDw8AvHxO5GVdQX9+vXv0HtNS0vFtWsl5uC/2f33T8T776/ByZMnLAJ4BwcHTJjwgPlrmUyGAQNCkJv7a4fuS0REZI1EIhFc1Aq4qBUY3K+lV315VX3TA6hu2Cx7Jr2lV72ro6IpqL/hIVTODnK2tewEggbwCoUCy5Ytu2U2fPPmza3GRo4caZGRv5mpU6di6tSpdzVHof02eL/deFeaODEWO3duw9GjB/H73z+BK1cu4+LFDDz11AIAQF1dLTZv/hj79u1BYWEBTKaWbj2VlZUduld+fh5CQ8Nbjfv6+rUau3QpExs3rsfZsz+iqqrK4lhVVcfuCzSVBbV1L7FYDB+f3sjP11uMe3h4tvplpFY7IjPzYofvTUREZCsc7eUYFOCGQQFu5rGqWgOu3tCnPiu/Aj//UgTTDdc0ld+01NW7OykZ1HcQdyHcAyO1Q+848/3fJ/6Fa3WlrcZdFM54acjCu51ah4SGhkOr7YVDh/bj979/AocO7QcAc+nIqlVvY9++PXj00ccxaFAoHBwcAIjwxhuvWQTznamiogJLljwDOzsHPP30QvTq5QO5XI6MjDSsX78WRmPXt4UUi9veod9V75mIiMha2StlGODnggF+Luax2voGZBdU3lCCU4nzl0tgNLX0qvf9zVNlPVxUEDOovykG8FbuocBYixp4AJCJZXgo8M5bNt6NCRMmYfPm/0NOTjaOHDmIoKAB5kx1c537kiUvm8+vq6vrcPYdADw9vZCTk91q/OrVLIuvf/opCWVlZVi+/G0MHtyyJ0Cvz23jVdv3i8DLS2u+142vaTKZkJOTjT59Or6hmoiIqKdSyqXo5+OMfj4tXdwMDY3IKayyqKs/fCbb/AAqhVwCPw8Hc+mNn5caWjc7SMTsVQ8wgLd6zRtVhe5C02zSpAexefP/4b33ViEnJ9siWG8rE71jx5dobGzs8H0iI6OwbdtWpKenmevgr127hkOHvrE4T3x9Id+Y7TYYDK3q5AFApVK164+J4OCBcHFxxa5d2/Hgg1PNTww+duwICgsLMHv23A6/HyIiImohk0rQR+uIPtqW1tQNjS0PoGpua3k8ORf1BuP1a8To7dFSetP8ACqZQE+qFxIDeBswwmsIRvsME+xJrDfq0ycAffv2R2LicYjFYowf37J5c/ToaBw4sA/29g7w9++D8+fP4cyZ03fUb/+JJ/6AAwf2YenSRZg58zEoFErs3h0HT08tKit/MZ8XGhoGtdoRy5e/gZkzZ0EkEuHAgX1oq3olKCgYBw9+g7Vr30Fw8ECoVHaIjh7T6jypVIrnnluCf/3rb1iy5FlMmDAJBQX52L79SwQEBGLatNadcIiIiOjuSCVi+F5vT4mwpjGj0YS8673qm+vqf0jNw7GfmhpFSMQi9NLYWwT1Ph4OUMi69wOoGMBTh02aFIuLFzMQETHU3I0GAF588RWIxWIcOvQN6urqERoajtWr38fSpUs6fA93d3e8++4GrFr1FjZv/tjiQU5vvvkP83lOTs54661VeO+91di4cT3UakdMmvQghg0bgaVLF1u85vTpv0NGRhr27duLL7/8HF5e2jYDeACYPHka5HI5tmz5BO+/vwb29vaYODEWCxcu4VNbiYiI7hGxWARvd3t4u9sjMsQLQNMDqIpKa5CV31JX/9MvRUjQNTWZEIkAbzd7iz71vp5qqBTdJ+wVmbjTrkOKiythNN78W5aXlwUvr9adUu6WVCq2igw8da6u+nnpyTQaNQoLK4SeBpHV41qh7sRkMuFaRZ1FTX1WfgVKK+vN53i6qMxZ+ubaegfVzR9A9f35POz8LhMl5XVwdVTgkfsCzX9EdDWxWAQ3N4ebHu8+f4oQERERUY8kEong6qiEq6MSEf015vGyyrqmTH1+Ba7mVeBSbjlOXygwH3dzVJqz9M3BvZODAt+fz8Mn36Sh/nrytLi8Dp98kwYA9yyIvxUG8ERERETULTk5KBDmoEBYYEuv+soaw28eQFWJsxmFN1wjR3VtAwy/qXyobzBi53eZDOCJiIiIiO4lB5UMA/1dMdDf1TxWU2fZq/5kSl6b1xaX192rad4SA3giIiIi6tFUCin693ZG/95NverTr15rM1h3c7SORhY9r3EmEREREdEtPHJfIOS/6S8vl4rxyH3W8TBHZuCJiIiIiG7QXOcuVBea22EA3wVMJhNEIpHQ0yArxw6uRERE1isyxAuRIV5W2XKVJTSdTCKRwmCov/2J1OMZDPWQSPg3NBEREXUMA/hO5uDgjNLSQtTX1zHDSm0ymUyor69DaWkhHBychZ4OERER2Rim/zqZSmUPACgrK0JjY0Onva5YLIbRyCexdhcSiRRqtYv554WIiIiovRjAdwGVyr7TAzNrrL8iIiIionuPJTRERERERDaEATwRERERkQ1hAE9EREREZEMYwBMRERER2RAG8ERERERENoRdaDpILBbuCatC3pvIlnCtELUP1wpR+9zrtXK7+4lMfNoQEREREZHNYAkNEREREZENYQBPRERERGRDGMATEREREdkQBvBERERERDaEATwRERERkQ1hAE9EREREZEMYwBMRERER2RAG8ERERERENoQBPBERERGRDWEAT0RERERkQ6RCT4DaVlBQgE8//RTJyclISUlBdXU1Pv30U4wcOVLoqRFZFZ1Oh7i4OJw6dQq5ublwdnZGREQEXnrpJfj5+Qk9PSKrce7cOfznP/9BamoqiouLoVarERwcjEWLFmHIkCFCT4/Iam3cuBErV65EcHAw4uPjhZ4OAAbwVuvy5cvYuHEj/Pz8EBQUhJ9++knoKRFZpU2bNuHs2bOIjY1FUFAQCgsLsWXLFsyYMQPbt29HYGCg0FMksgrZ2dlobGzEo48+Co1Gg4qKCuzZswdPPvkkNm7ciKioKKGnSGR1CgsLsX79etjZ2Qk9FQsik8lkEnoS1FplZSUMBgNcXFxw+PBhLFq0iBl4ojacPXsWgwYNglwuN49duXIF06ZNw5QpU/Dmm28KODsi61ZTU4MJEyZg0KBB2LBhg9DTIbI6f/nLX5CbmwuTyYTy8nKrycCzBt5KOTg4wMXFRehpEFm9IUOGWATvAODv749+/fohMzNToFkR2QaVSgVXV1eUl5cLPRUiq6PT6bB79268+uqrQk+lFQbwRNTtmEwmFBUV8Y9gojZUVlaipKQEly5dwjvvvIOMjAxERkYKPS0iq2IymfCPf/wDM2bMwIABA4SeTiusgSeibmf37t3Iz8/Hyy+/LPRUiKzOa6+9hgMHDgAAZDIZHnvsMSxcuFDgWRFZl127duHixYt4//33hZ5KmxjAE1G3kpmZib///e8YOnQopk+fLvR0iKzOokWLMGvWLOTl5SE+Ph719fUwGAytStGIeqrKykr8+9//xjPPPAMPDw+hp9MmltAQUbdRWFiIZ599Fk5OTlizZg3EYv6KI/qtoKAgREVF4Xe/+x0+/PBDnD9/3iprfImEsn79eshkMjz11FNCT+Wm+K8bEXULFRUVWLBgASoqKrBp0yZoNBqhp0Rk9WQyGcaPH4+DBw+itrZW6OkQCa6goACffPIJnnjiCRQVFSEnJwc5OTmoq6uDwWBATk4OysrKhJ4mS2iIyPbV1dVh4cKFuHLlCj7++GMEBAQIPSUim1FbWwuTyYSqqioolUqhp0MkqOLiYhgMBqxcuRIrV65sdXz8+PFYsGABXnnlFQFm14IBPBHZtMbGRrz00kv4+eefsW7dOgwePFjoKRFZpZKSEri6ulqMVVZW4sCBA9BqtXBzcxNoZkTWw8fHp82Nq6tXr0Z1dTVee+01+Pv73/uJ/QYDeCu2bt06ADD3so6Pj0dSUhIcHR3x5JNPCjk1Iqvx5ptv4ujRoxg3bhxKS0stHrJhb2+PCRMmCDg7Iuvx0ksvQaFQICIiAhqNBnq9Hjt37kReXh7eeecdoadHZBXUanWb/2588sknkEgkVvNvCp/EasWCgoLaHO/VqxeOHj16j2dDZJ3mzJmD06dPt3mMa4Woxfbt2xEfH4+LFy+ivLwcarUagwcPxrx58zBixAihp0dk1ebMmWNVT2JlAE9EREREZEPYhYaIiIiIyIYwgCciIiIisiEM4ImIiIiIbAgDeCIiIiIiG8IAnoiIiIjIhjCAJyIiIiKyIQzgiYiIiIhsCAN4IiKyenPmzMH9998v9DSIiKyCVOgJEBGRME6dOoW5c+fe9LhEIkFqauo9nBEREbUHA3gioh5u6tSpGDNmTKtxsZgf0hIRWSMG8EREPdzAgQMxffp0oadBRETtxPQKERHdUk5ODoKCgrB27Vrs3bsX06ZNQ2hoKMaOHYu1a9eioaGh1TVpaWlYtGgRRo4cidDQUEyePBkbN25EY2Njq3MLCwvxz3/+E+PHj8egQYMQGRmJp556CidOnGh1bn5+PpYuXYrhw4cjPDwcTz/9NC5fvtwl75uIyFoxA09E1MPV1NSgpKSk1bhcLoeDg4P566NHjyI7OxuzZ8+Gu7s7jh49ivfeew+5ublYsWKF+bxz585hzpw5kEql5nOPHTuGlStXIi0tDf/+97/N5+bk5ODxxx9HcXExpk+fjkGDBqGmpgbJyck4efIkoqKizOdWV1fjySefRHh4OF5++WXk5OTg008/xfPPP4+9e/dCIpF00XeIiMi6MIAnIurh1q5di7Vr17YaHzt2LDZs2GD+Oi0tDdu3b0dISAgA4Mknn8TixYuxc+dOzJo1C4MHDwYALF++HPX19di6dSuCg4PN57700kvYu3cvZs6cicjISADA3/72NxQUFGDTpk2IiYmxuL/RaLT4+tq1a3j66aexYMEC85irqyvefvttnDx5stX1RETdFQN4IqIebtasWYiNjW017urqavH16NGjzcE7AIhEIsyfPx+HDx/GoUOHMHjwYBQXF+Onn37CxIkTzcF787nPPfcc9u/fj0OHDiEyMhKlpaVISEhATExMm8H3bzfRisXiVl1zRo0aBQDIyspiAE9EPQYDeCKiHs7Pzw+jR4++7XmBgYGtxvr27QsAyM7OBtBUEnPj+I0CAgIgFovN5169ehUmkwkDBw5s1zw9PDygUCgsxpydnQEApaWl7XoNIqLugJtYiYjIJtyqxt1kMt3DmRARCYsBPBERtUtmZmarsYsXLwIAevfuDQDw8fGxGL/RpUuXYDQazef6+vpCJBLhwoULXTVlIqJuiQE8ERG1y8mTJ3H+/Hnz1yaTCZs2bQIATJgwAQDg5uaGiIgIHDt2DBkZGRbnfvDBBwCAiRMnAmgqfxkzZgyOHz+OkydPtrofs+pERG1jDTwRUQ+XmpqK+Pj4No81B+YAEBwcjD/84Q+YPXs2NBoNjhw5gpMnT2L69OmIiIgwn/f6669jzpw5mD17Np544gloNBocO3YMiYmJmDp1qrkDDQD8z//8D1JTU7FgwQLMmDEDISEhqKurQ3JyMnr16oU///nPXffGiYhsFAN4IqIebu/evdi7d2+bxw4ePGiuPb///vvRp08fbNiwAZcvX4abmxuef/55PP/88xbXhIaGYuvWrXj33XfxxRdfoLq6Gr1798Yrr7yCefPmWZzbu3dv7NixA++//z6OHz+O+Ph4ODo6Ijg4GLNmzeqaN0xEZONEJn5GSUREt5CTk4Px48dj8eLFWLJkidDTISLq8VgDT0RERERkQxjAExERERHZEAbwREREREQ2hDXwREREREQ2hBl4IiIiIiIbwgCeiIiIiMiGMIAnIiIiIrIhDOCJiIiIiGwIA3giIiIiIhvCAJ6IiIiIyIb8f88pujrAIXXXAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d5sCd-OmundD"
      },
      "source": [
        "# TODOs\n",
        "* Train and test on every language in PAWS-X\n",
        "* Implement weight decay\n",
        "* Fine-Tune on roBERTa, XLM-R."
      ]
    }
  ]
}