QuAC_BERT.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMQFUsOzZVXRaYw6MO2KXLi",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "f7e8c74965c04a2da66a8c960194fd65": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_104c7c0791e64f56ac31fec98a6a3558",
              "IPY_MODEL_d932f4fa727f4fc4997f9272dbdea64b",
              "IPY_MODEL_d97a9371945344f28d59441823f6d0c9"
            ],
            "layout": "IPY_MODEL_3d044ed3c9614c8db5849ab405643c6c"
          }
        },
        "104c7c0791e64f56ac31fec98a6a3558": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_acca7abcb1184458923feaf689c62354",
            "placeholder": "​",
            "style": "IPY_MODEL_d4817300a2904bef873db84ecbf7491f",
            "value": "100%"
          }
        },
        "d932f4fa727f4fc4997f9272dbdea64b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_aad20b3f78c245f987b617b868e67075",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1ceb371758d248c5b1bdfcf3acd24565",
            "value": 2
          }
        },
        "d97a9371945344f28d59441823f6d0c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2037e37d7c804062a1e1ac23cf51c7dd",
            "placeholder": "​",
            "style": "IPY_MODEL_dffcc5a7a8b543c9885d0d4c600f6a3b",
            "value": " 2/2 [00:00&lt;00:00, 53.15it/s]"
          }
        },
        "3d044ed3c9614c8db5849ab405643c6c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "acca7abcb1184458923feaf689c62354": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d4817300a2904bef873db84ecbf7491f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "aad20b3f78c245f987b617b868e67075": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1ceb371758d248c5b1bdfcf3acd24565": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2037e37d7c804062a1e1ac23cf51c7dd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dffcc5a7a8b543c9885d0d4c600f6a3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "59a0339411f345d9ac8ccba33efd1b00": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_dd8ae32c98fc47758160712e98de3cd4",
              "IPY_MODEL_2a6d6dd596a4495e85f2403c62bf4bfb",
              "IPY_MODEL_ebefb5cf60c94057a835abdb59987279"
            ],
            "layout": "IPY_MODEL_03b1e274f68d4ac28ace764359ec8b68"
          }
        },
        "dd8ae32c98fc47758160712e98de3cd4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3b81ff60558b4d4ca9e049b6d0d4b337",
            "placeholder": "​",
            "style": "IPY_MODEL_9bf0d5a08c02456aa450d40f46accc07",
            "value": "Map: 100%"
          }
        },
        "2a6d6dd596a4495e85f2403c62bf4bfb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_995d00e4d3974c978e50602d75d64435",
            "max": 11567,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_614c0045c052446fa7646c5fe5ae86fe",
            "value": 11567
          }
        },
        "ebefb5cf60c94057a835abdb59987279": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_08726ad8c89e458194688c6174b6d3e7",
            "placeholder": "​",
            "style": "IPY_MODEL_6ec78bc5fa1945bf8da420a9c1ef065e",
            "value": " 11567/11567 [00:37&lt;00:00, 425.68 examples/s]"
          }
        },
        "03b1e274f68d4ac28ace764359ec8b68": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "3b81ff60558b4d4ca9e049b6d0d4b337": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9bf0d5a08c02456aa450d40f46accc07": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "995d00e4d3974c978e50602d75d64435": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "614c0045c052446fa7646c5fe5ae86fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "08726ad8c89e458194688c6174b6d3e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6ec78bc5fa1945bf8da420a9c1ef065e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0a1efa3eaadb4025814238b045d35f14": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_de62eb77ae1d4189b7059a6ebfe3e65b",
              "IPY_MODEL_ab668a1f4b3e480b95dda0f77e2b41ce",
              "IPY_MODEL_cd66f3b2372b458fb11c98c37f5df2b7"
            ],
            "layout": "IPY_MODEL_83600bfae98e459588961834380266d7"
          }
        },
        "de62eb77ae1d4189b7059a6ebfe3e65b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f176d13668124094b4dd87cc088754a1",
            "placeholder": "​",
            "style": "IPY_MODEL_86886a257ddc4f18ba33464b732b788d",
            "value": "Map: 100%"
          }
        },
        "ab668a1f4b3e480b95dda0f77e2b41ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4440fe84cf2543a98443dbb95d3ab3b6",
            "max": 1000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a0924054d47f478493174f275bc012d0",
            "value": 1000
          }
        },
        "cd66f3b2372b458fb11c98c37f5df2b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ac9e9ad77f8848d48dc9caa1bedad893",
            "placeholder": "​",
            "style": "IPY_MODEL_0724730ba00a4a0da2575cb17a84cd7b",
            "value": " 1000/1000 [00:02&lt;00:00, 412.85 examples/s]"
          }
        },
        "83600bfae98e459588961834380266d7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "f176d13668124094b4dd87cc088754a1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "86886a257ddc4f18ba33464b732b788d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4440fe84cf2543a98443dbb95d3ab3b6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a0924054d47f478493174f275bc012d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ac9e9ad77f8848d48dc9caa1bedad893": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0724730ba00a4a0da2575cb17a84cd7b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "074ce09a5b4c437390c36f76cb4b0a5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4805b7c039ee44b6b09adfe0fc62a01d",
              "IPY_MODEL_f8d64cc2027a48b5856f59dbe3e0d4c8",
              "IPY_MODEL_fedc1a9c39304c81a62995d9ccf3d67f"
            ],
            "layout": "IPY_MODEL_51b848d340654868be463f7d53f32a39"
          }
        },
        "4805b7c039ee44b6b09adfe0fc62a01d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c9965ec2a61b4269a9f72a027cb355b2",
            "placeholder": "​",
            "style": "IPY_MODEL_72b68a2e0e1d4e8480147fd761557dc0",
            "value": "Map: 100%"
          }
        },
        "f8d64cc2027a48b5856f59dbe3e0d4c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fdc128e0e0c14af48c4f3f3f51a26041",
            "max": 100,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8e62703fbc2e40509dd0aad51b2a6e8a",
            "value": 100
          }
        },
        "fedc1a9c39304c81a62995d9ccf3d67f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d24f8767e9614e42ae9be93251c3daee",
            "placeholder": "​",
            "style": "IPY_MODEL_dcc4cbc9db3d42aba21096f039d45320",
            "value": " 100/100 [00:00&lt;00:00, 235.59 examples/s]"
          }
        },
        "51b848d340654868be463f7d53f32a39": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "c9965ec2a61b4269a9f72a027cb355b2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "72b68a2e0e1d4e8480147fd761557dc0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fdc128e0e0c14af48c4f3f3f51a26041": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8e62703fbc2e40509dd0aad51b2a6e8a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d24f8767e9614e42ae9be93251c3daee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dcc4cbc9db3d42aba21096f039d45320": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1cbbc4e8259a449a916c4a760a878dbc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d0195d5243c34953945b207ded9873d5",
              "IPY_MODEL_68d607e684d24132b145a9174b9bc917",
              "IPY_MODEL_86aa5fb8c2d847478a1afc3f7100ee17"
            ],
            "layout": "IPY_MODEL_c66d7d1531e44613b29e6afc36c503e0"
          }
        },
        "d0195d5243c34953945b207ded9873d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7a702624d7cb476c9f8b80fb9257de89",
            "placeholder": "​",
            "style": "IPY_MODEL_fbacdd25ec164dcb95b15410c4d30f55",
            "value": "Downloading pytorch_model.bin: 100%"
          }
        },
        "68d607e684d24132b145a9174b9bc917": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8c54eb5050cd49f18975759b5e4cac72",
            "max": 260793700,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_747900c595fc49f68a37f0e889ac74f6",
            "value": 260793700
          }
        },
        "86aa5fb8c2d847478a1afc3f7100ee17": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_aa52e4306c0a4d7791383ad9f020c6f0",
            "placeholder": "​",
            "style": "IPY_MODEL_3fb78cc5ea1a4708b20e01c6804d2624",
            "value": " 261M/261M [00:02&lt;00:00, 104MB/s]"
          }
        },
        "c66d7d1531e44613b29e6afc36c503e0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7a702624d7cb476c9f8b80fb9257de89": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fbacdd25ec164dcb95b15410c4d30f55": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8c54eb5050cd49f18975759b5e4cac72": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "747900c595fc49f68a37f0e889ac74f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "aa52e4306c0a4d7791383ad9f020c6f0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3fb78cc5ea1a4708b20e01c6804d2624": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "895b055707924d3ca79bf1d967bab680": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5f0b43f95bd94e40999be0f63b16b55f",
              "IPY_MODEL_c6fcd278b4214c5bb61e861e28673387",
              "IPY_MODEL_a15c25fca0994671b3fe507bc69b26fd"
            ],
            "layout": "IPY_MODEL_07781c574be142a4b93e77bb6c711129"
          }
        },
        "5f0b43f95bd94e40999be0f63b16b55f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_49ede41cad4f4cc49b86a5cb7a28af38",
            "placeholder": "​",
            "style": "IPY_MODEL_89b034aa0ab8414191e549ab032b5187",
            "value": "Downloading pytorch_model.bin: 100%"
          }
        },
        "c6fcd278b4214c5bb61e861e28673387": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_558e923a8cff41519c2a6bb6c8dc1f8b",
            "max": 435779157,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a929a3d39b584a7e8a6246f913665bb1",
            "value": 435779157
          }
        },
        "a15c25fca0994671b3fe507bc69b26fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4acdd85e47cf4ef0a2b521ca298bd56d",
            "placeholder": "​",
            "style": "IPY_MODEL_a7c8b48c662141878f1264e279def104",
            "value": " 436M/436M [00:04&lt;00:00, 111MB/s]"
          }
        },
        "07781c574be142a4b93e77bb6c711129": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "49ede41cad4f4cc49b86a5cb7a28af38": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "89b034aa0ab8414191e549ab032b5187": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "558e923a8cff41519c2a6bb6c8dc1f8b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a929a3d39b584a7e8a6246f913665bb1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4acdd85e47cf4ef0a2b521ca298bd56d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a7c8b48c662141878f1264e279def104": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "abdd211ddc9a482592d57ead925555bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "VBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "VBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "VBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d8dfbc85cc7e4654a2378a0a0d077383",
              "IPY_MODEL_a2c42cca09424120a78bb0ae77258e94",
              "IPY_MODEL_962dd21b0b88434ba5a3322cc58766fe",
              "IPY_MODEL_958de1ca3d2849169c69100d5483bb4e",
              "IPY_MODEL_31ef2a1cd9ce42c8a0047c55a1935a9d"
            ],
            "layout": "IPY_MODEL_fc3a80fc1fd54dd3871eed766d54d629"
          }
        },
        "d8dfbc85cc7e4654a2378a0a0d077383": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_32ae5bfad33142e7845a8cdf0fb6901b",
            "placeholder": "​",
            "style": "IPY_MODEL_01a2a6831af949bf933d5f26216a0f8a",
            "value": "<center> <img\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.svg\nalt='Hugging Face'> <br> Copy a token from <a\nhref=\"https://huggingface.co/settings/tokens\" target=\"_blank\">your Hugging Face\ntokens page</a> and paste it below. <br> Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file. </center>"
          }
        },
        "a2c42cca09424120a78bb0ae77258e94": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "PasswordModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "PasswordModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "PasswordView",
            "continuous_update": true,
            "description": "Token:",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_be00d7a366394e9dbb0cf9149051aa35",
            "placeholder": "​",
            "style": "IPY_MODEL_d8a84a0d44164b3cb1ea3c9724a7713a",
            "value": ""
          }
        },
        "962dd21b0b88434ba5a3322cc58766fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "CheckboxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "CheckboxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "CheckboxView",
            "description": "Add token as git credential?",
            "description_tooltip": null,
            "disabled": false,
            "indent": true,
            "layout": "IPY_MODEL_f3f23aadd37b4f78adc5699faf956d24",
            "style": "IPY_MODEL_df52a0c2d2424eaf9eab3652a8d18510",
            "value": false
          }
        },
        "958de1ca3d2849169c69100d5483bb4e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ButtonView",
            "button_style": "",
            "description": "Login",
            "disabled": false,
            "icon": "",
            "layout": "IPY_MODEL_0bd7b31a5641402c853762881eb95beb",
            "style": "IPY_MODEL_76558554460c48d28c0a27aa9a0acf18",
            "tooltip": ""
          }
        },
        "31ef2a1cd9ce42c8a0047c55a1935a9d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f7ae6d00cc874651991dcc0558afac6b",
            "placeholder": "​",
            "style": "IPY_MODEL_cfb3246c789f47b4bf0920d03683f087",
            "value": "\n<b>Pro Tip:</b> If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks. </center>"
          }
        },
        "fc3a80fc1fd54dd3871eed766d54d629": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": "center",
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": "flex",
            "flex": null,
            "flex_flow": "column",
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "32ae5bfad33142e7845a8cdf0fb6901b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "01a2a6831af949bf933d5f26216a0f8a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "be00d7a366394e9dbb0cf9149051aa35": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d8a84a0d44164b3cb1ea3c9724a7713a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f3f23aadd37b4f78adc5699faf956d24": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "df52a0c2d2424eaf9eab3652a8d18510": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0bd7b31a5641402c853762881eb95beb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "76558554460c48d28c0a27aa9a0acf18": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "button_color": null,
            "font_weight": ""
          }
        },
        "f7ae6d00cc874651991dcc0558afac6b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cfb3246c789f47b4bf0920d03683f087": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/SRDdev/QABERT-small/blob/master/QuAC_BERT.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "###⬇️ Imports & Installations\n",
        "\n",
        "In this section we will import and install all the required libraries."
      ],
      "metadata": {
        "id": "PTFqJyB9QqYi"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "c2UsJCiR8FCK"
      },
      "outputs": [],
      "source": [
        "! pip install transformers datasets"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Load Dataset\n",
        "We will be using the QuAC dataset from Huggingface Datasets library.[Link](https://huggingface.co/datasets/quac)"
      ],
      "metadata": {
        "id": "ua4XqZC8UPqT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_dataset , Dataset\n",
        "raw_datasets  = load_dataset(\"quac\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86,
          "referenced_widgets": [
            "f7e8c74965c04a2da66a8c960194fd65",
            "104c7c0791e64f56ac31fec98a6a3558",
            "d932f4fa727f4fc4997f9272dbdea64b",
            "d97a9371945344f28d59441823f6d0c9",
            "3d044ed3c9614c8db5849ab405643c6c",
            "acca7abcb1184458923feaf689c62354",
            "d4817300a2904bef873db84ecbf7491f",
            "aad20b3f78c245f987b617b868e67075",
            "1ceb371758d248c5b1bdfcf3acd24565",
            "2037e37d7c804062a1e1ac23cf51c7dd",
            "dffcc5a7a8b543c9885d0d4c600f6a3b"
          ]
        },
        "id": "u1P9BqCl8N0X",
        "outputId": "30f11301-113f-40cd-a82d-b2d7623580db"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:datasets.builder:Found cached dataset quac (/root/.cache/huggingface/datasets/quac/plain_text/1.1.0/4170258e7e72d7c81bd6441b3f3489ea1544f0ff226ce61e22bb00c6e9d01fb6)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f7e8c74965c04a2da66a8c960194fd65"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "raw_datasets "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xhVIEzm78Tf-",
        "outputId": "4964f24d-b64e-457f-fd4c-a7d1f7722728"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['dialogue_id', 'wikipedia_page_title', 'background', 'section_title', 'context', 'turn_ids', 'questions', 'followups', 'yesnos', 'answers', 'orig_answers'],\n",
              "        num_rows: 11567\n",
              "    })\n",
              "    validation: Dataset({\n",
              "        features: ['dialogue_id', 'wikipedia_page_title', 'background', 'section_title', 'context', 'turn_ids', 'questions', 'followups', 'yesnos', 'answers', 'orig_answers'],\n",
              "        num_rows: 1000\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Lets check what each coolumn(feature) contains"
      ],
      "metadata": {
        "id": "y8cjUwSWUd3h"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"id: \", raw_datasets[\"train\"][0][\"dialogue_id\"][len(raw_datasets[\"train\"][0][\"dialogue_id\"])-1:])\n",
        "print(\"Context: \", raw_datasets[\"train\"][0][\"context\"])\n",
        "print(\"Question: \", raw_datasets[\"train\"][0][\"questions\"])\n",
        "print(\"Answer: \", raw_datasets[\"train\"][0][\"answers\"]['texts'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dwbc1iip8WYW",
        "outputId": "540e799f-c414-4ae7-ff6f-8d2a78e06e4a"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "id:  1\n",
            "Context:  According to the Indian census of 2001, there were 30,803,747 speakers of Malayalam in Kerala, making up 93.2% of the total number of Malayalam speakers in India, and 96.7% of the total population of the state. There were a further 701,673 (2.1% of the total number) in Karnataka, 557,705 (1.7%) in Tamil Nadu and 406,358 (1.2%) in Maharashtra. The number of Malayalam speakers in Lakshadweep is 51,100, which is only 0.15% of the total number, but is as much as about 84% of the population of Lakshadweep. In all, Malayalis made up 3.22% of the total Indian population in 2001. Of the total 33,066,392 Malayalam speakers in India in 2001, 33,015,420 spoke the standard dialects, 19,643 spoke the Yerava dialect and 31,329 spoke non-standard regional variations like Eranadan. As per the 1991 census data, 28.85% of all Malayalam speakers in India spoke a second language and 19.64% of the total knew three or more languages.  Large numbers of Malayalis have settled in Bangalore, Mangalore, Delhi, Coimbatore, Hyderabad, Mumbai (Bombay), Ahmedabad, Pune, and Chennai (Madras). A large number of Malayalis have also emigrated to the Middle East, the United States, and Europe. Accessed November 22, 2014.</ref> including a large number of professionals. There were 7,093 Malayalam speakers in Australia in 2006. The 2001 Canadian census reported 7,070 people who listed Malayalam as their mother tongue, mostly in the Greater Toronto Area and Southern Ontario. In 2010, the Census of Population of Singapore reported that there were 26,348 Malayalees in Singapore. The 2006 New Zealand census reported 2,139 speakers. 134 Malayalam speaking households were reported in 1956 in Fiji. There is also a considerable Malayali population in the Persian Gulf regions, especially in Bahrain, Muscat, Doha, Dubai, Abu Dhabi, Kuwait and European region mainly in London.  World Malayalee Council, the organisation working with the Malayali diaspora across the Globe has embarked upon a project for making a data bank of the diaspora. CANNOTANSWER\n",
            "Question:  ['Where is Malayali located?', 'What other languages are spoken there?', 'What else is this place known for?', 'Were they ever successful in doing this?', 'Do they produce anything from here?', 'Is this population still growing?', 'Is the country thriving?']\n",
            "Answer:  [['30,803,747 speakers of Malayalam in Kerala, making up 93.2% of the total number of Malayalam speakers in India,'], ['33,015,420 spoke the standard dialects, 19,643 spoke the Yerava dialect and 31,329 spoke non-standard regional variations like Eranadan.'], ['World Malayalee Council, the organisation working with the Malayali diaspora across the Globe has embarked upon a project for making a data bank of the diaspora.'], ['CANNOTANSWER'], ['CANNOTANSWER'], ['In 2010, the Census of Population of Singapore reported that there were 26,348 Malayalees in Singapore.'], ['CANNOTANSWER']]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We will now create our own dataset , and remove the unwantted stuff like `\"wikipedia_page_title\"` etc"
      ],
      "metadata": {
        "id": "qzBAEQBxUjpJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Loop through each example in the 'train' split\n",
        "question_train = []\n",
        "answer_train = []\n",
        "answer_starts_t = []\n",
        "context_train = []\n",
        "for example in raw_datasets['train']:\n",
        "    context_train.append(example['context'])\n",
        "    answer_train.append(example['answers']['texts'][0])\n",
        "    answer_starts_t.append(example[\"answers\"]['answer_starts'][0])\n",
        "    question_train.append(example['questions'][0])\n",
        "# Loop through each example in the 'val' split\n",
        "question_val = []\n",
        "answer_val = []\n",
        "answer_starts_v=[]\n",
        "context_val = []\n",
        "for example in raw_datasets['validation']:\n",
        "    context_val.append(example['context'])\n",
        "    answer_val.append(example['answers']['texts'])\n",
        "    answer_starts_v.append(example[\"answers\"]['answer_starts'][0])\n",
        "    question_val.append(example['questions'][0])"
      ],
      "metadata": {
        "id": "WwRnFLQq_M6S"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#ID\n",
        "id_t = []\n",
        "id_v = []\n",
        "for i in range(len(raw_datasets['train'])):\n",
        "  id_t.append(i)\n",
        "for i in range(len(raw_datasets['validation'])):\n",
        "  id_v.append(i)"
      ],
      "metadata": {
        "id": "ezMlUEafOSsq"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "answer_t = Dataset.from_dict({\"texts\":answer_train,\"answer_starts\":answer_starts_t})\n",
        "answer_v = Dataset.from_dict({\"texts\":answer_val,\"answer_starts\":answer_starts_v})\n",
        "\n",
        "train_data = {\"question\": question_train,\"answer\":answer_t,\"context\": context_train,\"id\":id_t}\n",
        "val_data = {\"question\": question_val,\"answer\":answer_v,\"context\": context_val,\"id\":id_v}\n",
        "\n",
        "train_dataset = Dataset.from_dict(train_data)\n",
        "val_dataset = Dataset.from_dict(val_data)\n",
        "\n",
        "data = {\n",
        "    \"train\":train_dataset,\n",
        "    \"val\":val_dataset\n",
        "}\n",
        "data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cbGjm-OkBdah",
        "outputId": "61299260-c557-4492-e287-44f7c494ef6d"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'train': Dataset({\n",
              "     features: ['question', 'answer', 'context', 'id'],\n",
              "     num_rows: 11567\n",
              " }),\n",
              " 'val': Dataset({\n",
              "     features: ['question', 'answer', 'context', 'id'],\n",
              "     num_rows: 1000\n",
              " })}"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### ⚒️ Preprocessing\n",
        "Now that we have our HF dataset we can move to the preprocessing part. We will tokenize the dataset using pretrianed tokenizers from HF. \n",
        "\n",
        "The toknizer which we will use is called `BERT-BASE-CASED`"
      ],
      "metadata": {
        "id": "58K-r5UpUwwK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer\n",
        "\n",
        "model_checkpoint = \"bert-base-cased\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)"
      ],
      "metadata": {
        "id": "bzQrAUBXEbNM"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "context = data[\"train\"][0][\"context\"]\n",
        "question = data[\"train\"][0][\"question\"]\n",
        "\n",
        "inputs = tokenizer(question, context)\n",
        "tokenizer.decode(inputs[\"input_ids\"])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 244
        },
        "id": "W3jvW_khFKF6",
        "outputId": "c369076b-30c3-4d8c-f910-7b50a71b4f03"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'[CLS] Where is Malayali located? [SEP] According to the Indian census of 2001, there were 30, 803, 747 speakers of Malayalam in Kerala, making up 93. 2 % of the total number of Malayalam speakers in India, and 96. 7 % of the total population of the state. There were a further 701, 673 ( 2. 1 % of the total number ) in Karnataka, 557, 705 ( 1. 7 % ) in Tamil Nadu and 406, 358 ( 1. 2 % ) in Maharashtra. The number of Malayalam speakers in Lakshadweep is 51, 100, which is only 0. 15 % of the total number, but is as much as about 84 % of the population of Lakshadweep. In all, Malayalis made up 3. 22 % of the total Indian population in 2001. Of the total 33, 066, 392 Malayalam speakers in India in 2001, 33, 015, 420 spoke the standard dialects, 19, 643 spoke the Yerava dialect and 31, 329 spoke non - standard regional variations like Eranadan. As per the 1991 census data, 28. 85 % of all Malayalam speakers in India spoke a second language and 19. 64 % of the total knew three or more languages. Large numbers of Malayalis have settled in Bangalore, Mangalore, Delhi, Coimbatore, Hyderabad, Mumbai ( Bombay ), Ahmedabad, Pune, and Chennai ( Madras ). A large number of Malayalis have also emigrated to the Middle East, the United States, and Europe. Accessed November 22, 2014. < / ref > including a large number of professionals. There were 7, 093 Malayalam speakers in Australia in 2006. The 2001 Canadian census reported 7, 070 people who listed Malayalam as their mother tongue, mostly in the Greater Toronto Area and Southern Ontario. In 2010, the Census of Population of Singapore reported that there were 26, 348 Malayalees in Singapore. The 2006 New Zealand census reported 2, 139 speakers. 134 Malayalam speaking households were reported in 1956 in Fiji. There is also a considerable Malayali population in the Persian Gulf regions, especially in Bahrain, Muscat, Doha, Dubai, Abu Dhabi, Kuwait and European region mainly in London. World Malayalee Council, the organisation working with the Malayali diaspora across the Globe has embarked upon a project for making a data bank of the diaspora. CANNOTANSWER [SEP]'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "inputs = tokenizer(\n",
        "    question,\n",
        "    context,\n",
        "    max_length=100,\n",
        "    truncation=\"only_second\",\n",
        "    stride=50,\n",
        "    return_overflowing_tokens=True,\n",
        ")\n",
        "\n",
        "for ids in inputs[\"input_ids\"]:\n",
        "    print(tokenizer.decode(ids))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZfIXJzfvFBrH",
        "outputId": "1a798f25-0fba-4a24-87c9-69dba69cd18c"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CLS] Where is Malayali located? [SEP] According to the Indian census of 2001, there were 30, 803, 747 speakers of Malayalam in Kerala, making up 93. 2 % of the total number of Malayalam speakers in India, and 96. 7 % of the total population of the state. There were a further 701, 673 ( 2. 1 % of the total number ) in Karnataka, 557, 705 ( 1. 7 % ) in Tamil Nadu and 406 [SEP]\n",
            "[CLS] Where is Malayali located? [SEP]. 7 % of the total population of the state. There were a further 701, 673 ( 2. 1 % of the total number ) in Karnataka, 557, 705 ( 1. 7 % ) in Tamil Nadu and 406, 358 ( 1. 2 % ) in Maharashtra. The number of Malayalam speakers in Lakshadweep is 51, 100, which is only 0. 15 % of the total number, but [SEP]\n",
            "[CLS] Where is Malayali located? [SEP] 7 % ) in Tamil Nadu and 406, 358 ( 1. 2 % ) in Maharashtra. The number of Malayalam speakers in Lakshadweep is 51, 100, which is only 0. 15 % of the total number, but is as much as about 84 % of the population of Lakshadweep. In all, Malayalis made up 3. 22 % of the total Indian population in 2001. Of the total 33, [SEP]\n",
            "[CLS] Where is Malayali located? [SEP]. 15 % of the total number, but is as much as about 84 % of the population of Lakshadweep. In all, Malayalis made up 3. 22 % of the total Indian population in 2001. Of the total 33, 066, 392 Malayalam speakers in India in 2001, 33, 015, 420 spoke the standard dialects, 19, 643 spoke the Yerava dialect and 31, 329 spoke non - [SEP]\n",
            "[CLS] Where is Malayali located? [SEP] population in 2001. Of the total 33, 066, 392 Malayalam speakers in India in 2001, 33, 015, 420 spoke the standard dialects, 19, 643 spoke the Yerava dialect and 31, 329 spoke non - standard regional variations like Eranadan. As per the 1991 census data, 28. 85 % of all Malayalam speakers in India spoke a second language and 19. 64 % of the total knew three or more [SEP]\n",
            "[CLS] Where is Malayali located? [SEP] dialect and 31, 329 spoke non - standard regional variations like Eranadan. As per the 1991 census data, 28. 85 % of all Malayalam speakers in India spoke a second language and 19. 64 % of the total knew three or more languages. Large numbers of Malayalis have settled in Bangalore, Mangalore, Delhi, Coimbatore, Hyderabad, Mumbai ( Bombay ), Ahmedabad, Pune, and Chennai ( Madras ). [SEP]\n",
            "[CLS] Where is Malayali located? [SEP] 64 % of the total knew three or more languages. Large numbers of Malayalis have settled in Bangalore, Mangalore, Delhi, Coimbatore, Hyderabad, Mumbai ( Bombay ), Ahmedabad, Pune, and Chennai ( Madras ). A large number of Malayalis have also emigrated to the Middle East, the United States, and Europe. Accessed November 22, 2014. < / ref > including a large number of professionals. There [SEP]\n",
            "[CLS] Where is Malayali located? [SEP], Pune, and Chennai ( Madras ). A large number of Malayalis have also emigrated to the Middle East, the United States, and Europe. Accessed November 22, 2014. < / ref > including a large number of professionals. There were 7, 093 Malayalam speakers in Australia in 2006. The 2001 Canadian census reported 7, 070 people who listed Malayalam as their mother tongue, mostly in the Greater Toronto Area and Southern Ontario. In [SEP]\n",
            "[CLS] Where is Malayali located? [SEP] > including a large number of professionals. There were 7, 093 Malayalam speakers in Australia in 2006. The 2001 Canadian census reported 7, 070 people who listed Malayalam as their mother tongue, mostly in the Greater Toronto Area and Southern Ontario. In 2010, the Census of Population of Singapore reported that there were 26, 348 Malayalees in Singapore. The 2006 New Zealand census reported 2, 139 speakers. 134 Malayalam speaking households were reported in 1956 [SEP]\n",
            "[CLS] Where is Malayali located? [SEP] the Greater Toronto Area and Southern Ontario. In 2010, the Census of Population of Singapore reported that there were 26, 348 Malayalees in Singapore. The 2006 New Zealand census reported 2, 139 speakers. 134 Malayalam speaking households were reported in 1956 in Fiji. There is also a considerable Malayali population in the Persian Gulf regions, especially in Bahrain, Muscat, Doha, Dubai, Abu Dhabi, Kuwait and European region mainly in London. [SEP]\n",
            "[CLS] Where is Malayali located? [SEP]. 134 Malayalam speaking households were reported in 1956 in Fiji. There is also a considerable Malayali population in the Persian Gulf regions, especially in Bahrain, Muscat, Doha, Dubai, Abu Dhabi, Kuwait and European region mainly in London. World Malayalee Council, the organisation working with the Malayali diaspora across the Globe has embarked upon a project for making a data bank of the diaspora. CANNOTANSWER [SEP]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "inputs = tokenizer(\n",
        "    question,\n",
        "    context,\n",
        "    max_length=100,\n",
        "    truncation=\"only_second\",\n",
        "    stride=50,\n",
        "    return_overflowing_tokens=True,\n",
        "    return_offsets_mapping=True,\n",
        ")\n",
        "inputs.keys()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DFowcodKFGCA",
        "outputId": "15b0412e-cf31-43fe-fe77-db81f21f4f34"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['input_ids', 'token_type_ids', 'attention_mask', 'offset_mapping', 'overflow_to_sample_mapping'])"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "inputs = tokenizer(\n",
        "    data[\"train\"][2:6][\"question\"],\n",
        "    data[\"train\"][2:6][\"context\"],\n",
        "    max_length=100,\n",
        "    truncation=\"only_second\",\n",
        "    stride=50,\n",
        "    return_overflowing_tokens=True,\n",
        "    return_offsets_mapping=True,\n",
        ")\n",
        "\n",
        "print(f\"The 4 examples gave {len(inputs['input_ids'])} features.\")\n",
        "print(f\"Here is where each comes from: {inputs['overflow_to_sample_mapping']}.\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c_0TKb5PFm0L",
        "outputId": "283d03db-6f11-40ec-b2ab-558176dcf612"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The 4 examples gave 43 features.\n",
            "Here is where each comes from: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3].\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_data = data[\"train\"]\n",
        "answers = train_data[2:6][\"answer\"]\n",
        "start_positions = []\n",
        "end_positions = []\n",
        "\n",
        "for i, offset in enumerate(inputs[\"offset_mapping\"]):\n",
        "    sample_idx = inputs[\"overflow_to_sample_mapping\"][i]\n",
        "    answer = answers[sample_idx]\n",
        "    start_char = answer[\"answer_starts\"][0]  # Update field name\n",
        "    end_char = answer[\"answer_starts\"][0] + len(answer[\"texts\"][0])  # Update field name\n",
        "    sequence_ids = inputs.sequence_ids(i)\n",
        "\n",
        "    # Find the start and end of the context\n",
        "    idx = 0\n",
        "    while sequence_ids[idx] != 1:\n",
        "        idx += 1\n",
        "    context_start = idx\n",
        "    while sequence_ids[idx] == 1:\n",
        "        idx += 1\n",
        "    context_end = idx - 1\n",
        "\n",
        "    # If the answer is not fully inside the context, label is (0, 0)\n",
        "    if offset[context_start][0] > start_char or offset[context_end][1] < end_char:\n",
        "        start_positions.append(0)\n",
        "        end_positions.append(0)\n",
        "    else:\n",
        "        # Otherwise it's the start and end token positions\n",
        "        idx = context_start\n",
        "        while idx <= context_end and offset[idx][0] <= start_char:\n",
        "            idx += 1\n",
        "        start_positions.append(idx - 1)\n",
        "\n",
        "        idx = context_end\n",
        "        while idx >= context_start and offset[idx][1] >= end_char:\n",
        "            idx -= 1\n",
        "        end_positions.append(idx + 1)\n",
        "\n",
        "start_positions, end_positions\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xRFoRKpYH6-A",
        "outputId": "d023ad0f-0121-409c-abdd-07a2584de37d"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "([25,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  9,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  67,\n",
              "  28,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  22,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0],\n",
              " [41,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  30,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  74,\n",
              "  35,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  47,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0])"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "idx = 0\n",
        "sample_idx = inputs[\"overflow_to_sample_mapping\"][idx]\n",
        "answer = answers[sample_idx][\"texts\"][0]\n",
        "\n",
        "start = start_positions[idx]\n",
        "end = end_positions[idx]\n",
        "labeled_answer = tokenizer.decode(inputs[\"input_ids\"][idx][start : end + 1])\n",
        "\n",
        "print(f\"Theoretical answer: {answer}, labels give: {labeled_answer}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pvGn7-JrJZpM",
        "outputId": "2747d12a-db17-4575-b406-9e336ccd7479"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Theoretical answer: 19-year-old Cove Reber was announced as their new permanent lead singer., labels give: 19 - year - old Cove Reber was announced as their new permanent lead singer.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "idx = 4\n",
        "sample_idx = inputs[\"overflow_to_sample_mapping\"][idx]\n",
        "answer = answers[sample_idx][\"texts\"][0]\n",
        "\n",
        "decoded_example = tokenizer.decode(inputs[\"input_ids\"][idx])\n",
        "print(f\"Theoretical answer: {answer}, decoded example: {decoded_example}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Da7mux2SL6a4",
        "outputId": "3d866456-887e-4106-a78c-89119bddb7d6"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Theoretical answer: 19-year-old Cove Reber was announced as their new permanent lead singer., decoded example: [CLS] What do we know about Cove Reber? [SEP] their lives... Saosin is a band on a completely different level. All these dudes are freaks about music. \" Reber's addition to the band was difficult, for the more experienced Green was the center piece of the band in the eyes of Saosin's fans. Many fans consider the time with Green to be something entirely different from the time with Reber. There are still distinct fans of both [SEP]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Preprocessing Training data"
      ],
      "metadata": {
        "id": "IGVj6V-jVJJZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "max_length = 384\n",
        "stride = 128\n",
        "\n",
        "def preprocess_training_examples(examples):\n",
        "    questions = [q.strip() for q in examples[\"question\"]]\n",
        "    inputs = tokenizer(\n",
        "        questions,\n",
        "        examples[\"context\"],\n",
        "        max_length=max_length,\n",
        "        truncation=\"only_second\",\n",
        "        stride=stride,\n",
        "        return_overflowing_tokens=True,\n",
        "        return_offsets_mapping=True,\n",
        "        padding=\"max_length\",\n",
        "    )\n",
        "\n",
        "    offset_mapping = inputs.pop(\"offset_mapping\")\n",
        "    sample_map = inputs.pop(\"overflow_to_sample_mapping\")\n",
        "    answers = examples[\"answer\"]\n",
        "    start_positions = []\n",
        "    end_positions = []\n",
        "\n",
        "    for i, offset in enumerate(offset_mapping):\n",
        "        sample_idx = sample_map[i]\n",
        "        answer = answers[sample_idx]\n",
        "        start_char = answer[\"answer_starts\"][0]\n",
        "        end_char = answer[\"answer_starts\"][0] + len(answer[\"texts\"][0])\n",
        "        sequence_ids = inputs.sequence_ids(i)\n",
        "\n",
        "        # Find the start and end of the context\n",
        "        idx = 0\n",
        "        while sequence_ids[idx] != 1:\n",
        "            idx += 1\n",
        "        context_start = idx\n",
        "        while sequence_ids[idx] == 1:\n",
        "            idx += 1\n",
        "        context_end = idx - 1\n",
        "\n",
        "        # If the answer is not fully inside the context, label is (0, 0)\n",
        "        if offset[context_start][0] > start_char or offset[context_end][1] < end_char:\n",
        "            start_positions.append(0)\n",
        "            end_positions.append(0)\n",
        "        else:\n",
        "            # Otherwise it's the start and end token positions\n",
        "            idx = context_start\n",
        "            while idx <= context_end and offset[idx][0] <= start_char:\n",
        "                idx += 1\n",
        "            start_positions.append(idx - 1)\n",
        "\n",
        "            idx = context_end\n",
        "            while idx >= context_start and offset[idx][1] >= end_char:\n",
        "                idx -= 1\n",
        "            end_positions.append(idx + 1)\n",
        "\n",
        "    inputs[\"start_positions\"] = start_positions\n",
        "    inputs[\"end_positions\"] = end_positions\n",
        "    return inputs\n"
      ],
      "metadata": {
        "id": "1poHqmhGL-RC"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_dataset = data[\"train\"].map(\n",
        "    preprocess_training_examples,\n",
        "    batched=True,\n",
        "    remove_columns=data[\"train\"].column_names,\n",
        ")\n",
        "len(data[\"train\"]), len(train_dataset)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34,
          "referenced_widgets": [
            "59a0339411f345d9ac8ccba33efd1b00",
            "dd8ae32c98fc47758160712e98de3cd4",
            "2a6d6dd596a4495e85f2403c62bf4bfb",
            "ebefb5cf60c94057a835abdb59987279",
            "03b1e274f68d4ac28ace764359ec8b68",
            "3b81ff60558b4d4ca9e049b6d0d4b337",
            "9bf0d5a08c02456aa450d40f46accc07",
            "995d00e4d3974c978e50602d75d64435",
            "614c0045c052446fa7646c5fe5ae86fe",
            "08726ad8c89e458194688c6174b6d3e7",
            "6ec78bc5fa1945bf8da420a9c1ef065e"
          ]
        },
        "id": "DKdRN98AMBZF",
        "outputId": "7d6ca802-0322-4b86-d72d-52689206d4d4"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/11567 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "59a0339411f345d9ac8ccba33efd1b00"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(11567, 25362)"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Preprocessing Validation Data"
      ],
      "metadata": {
        "id": "7YPpG80qVM_p"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def preprocess_validation_examples(examples):\n",
        "    questions = [q.strip() for q in examples[\"question\"]]\n",
        "    inputs = tokenizer(\n",
        "        questions,\n",
        "        examples[\"context\"],\n",
        "        max_length=max_length,\n",
        "        truncation=\"only_second\",\n",
        "        stride=stride,\n",
        "        return_overflowing_tokens=True,\n",
        "        return_offsets_mapping=True,\n",
        "        padding=\"max_length\",\n",
        "    )\n",
        "\n",
        "    sample_map = inputs.pop(\"overflow_to_sample_mapping\")\n",
        "    example_ids = []\n",
        "\n",
        "    for i in range(len(inputs[\"input_ids\"])):\n",
        "        sample_idx = sample_map[i]\n",
        "        example_ids.append(examples[\"id\"][sample_idx])\n",
        "\n",
        "        sequence_ids = inputs.sequence_ids(i)\n",
        "        offset = inputs[\"offset_mapping\"][i]\n",
        "        inputs[\"offset_mapping\"][i] = [\n",
        "            o if sequence_ids[k] == 1 else None for k, o in enumerate(offset)\n",
        "        ]\n",
        "\n",
        "    inputs[\"example_id\"] = example_ids\n",
        "    return inputs"
      ],
      "metadata": {
        "id": "azCYvyvgMC73"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "validation_dataset = data[\"val\"].map(\n",
        "    preprocess_validation_examples,\n",
        "    batched=True,\n",
        "    remove_columns=data[\"val\"].column_names,\n",
        ")\n",
        "len(data[\"val\"]), len(validation_dataset)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34,
          "referenced_widgets": [
            "0a1efa3eaadb4025814238b045d35f14",
            "de62eb77ae1d4189b7059a6ebfe3e65b",
            "ab668a1f4b3e480b95dda0f77e2b41ce",
            "cd66f3b2372b458fb11c98c37f5df2b7",
            "83600bfae98e459588961834380266d7",
            "f176d13668124094b4dd87cc088754a1",
            "86886a257ddc4f18ba33464b732b788d",
            "4440fe84cf2543a98443dbb95d3ab3b6",
            "a0924054d47f478493174f275bc012d0",
            "ac9e9ad77f8848d48dc9caa1bedad893",
            "0724730ba00a4a0da2575cb17a84cd7b"
          ]
        },
        "id": "dvZVWBW3M9EJ",
        "outputId": "ed3f6a58-e0cf-44e5-b0ec-aef31572443a"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/1000 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0a1efa3eaadb4025814238b045d35f14"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1000, 2452)"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "small_eval_set = data[\"val\"].select(range(100))\n",
        "trained_checkpoint = \"distilbert-base-cased-distilled-squad\"\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(trained_checkpoint)\n",
        "eval_set = small_eval_set.map(\n",
        "    preprocess_validation_examples,\n",
        "    batched=True,\n",
        "    remove_columns=data[\"val\"].column_names,\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "074ce09a5b4c437390c36f76cb4b0a5e",
            "4805b7c039ee44b6b09adfe0fc62a01d",
            "f8d64cc2027a48b5856f59dbe3e0d4c8",
            "fedc1a9c39304c81a62995d9ccf3d67f",
            "51b848d340654868be463f7d53f32a39",
            "c9965ec2a61b4269a9f72a027cb355b2",
            "72b68a2e0e1d4e8480147fd761557dc0",
            "fdc128e0e0c14af48c4f3f3f51a26041",
            "8e62703fbc2e40509dd0aad51b2a6e8a",
            "d24f8767e9614e42ae9be93251c3daee",
            "dcc4cbc9db3d42aba21096f039d45320"
          ]
        },
        "id": "KcIPEI5KNDm4",
        "outputId": "cea1af55-0297-4f22-df5d-d20067b1ba72"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/100 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "074ce09a5b4c437390c36f76cb4b0a5e"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)"
      ],
      "metadata": {
        "id": "2R-KW7EaPYbp"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "from transformers import AutoModelForQuestionAnswering\n",
        "\n",
        "eval_set_for_model = eval_set.remove_columns([\"example_id\", \"offset_mapping\"])\n",
        "eval_set_for_model.set_format(\"torch\")\n",
        "\n",
        "device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
        "batch = {k: eval_set_for_model[k].to(device) for k in eval_set_for_model.column_names}\n",
        "trained_model = AutoModelForQuestionAnswering.from_pretrained(trained_checkpoint).to(\n",
        "    device\n",
        ")\n",
        "\n",
        "with torch.no_grad():\n",
        "    outputs = trained_model(**batch)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "1cbbc4e8259a449a916c4a760a878dbc",
            "d0195d5243c34953945b207ded9873d5",
            "68d607e684d24132b145a9174b9bc917",
            "86aa5fb8c2d847478a1afc3f7100ee17",
            "c66d7d1531e44613b29e6afc36c503e0",
            "7a702624d7cb476c9f8b80fb9257de89",
            "fbacdd25ec164dcb95b15410c4d30f55",
            "8c54eb5050cd49f18975759b5e4cac72",
            "747900c595fc49f68a37f0e889ac74f6",
            "aa52e4306c0a4d7791383ad9f020c6f0",
            "3fb78cc5ea1a4708b20e01c6804d2624"
          ]
        },
        "id": "fYZv1Ni9PeZZ",
        "outputId": "1deac3f4-dfda-4850-dcb3-6431b77d77f0"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading pytorch_model.bin:   0%|          | 0.00/261M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1cbbc4e8259a449a916c4a760a878dbc"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "start_logits = outputs.start_logits.cpu().numpy()\n",
        "end_logits = outputs.end_logits.cpu().numpy()"
      ],
      "metadata": {
        "id": "z7Cz5alOPfwZ"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import collections\n",
        "\n",
        "example_to_features = collections.defaultdict(list)\n",
        "for idx, feature in enumerate(eval_set):\n",
        "    example_to_features[feature[\"example_id\"]].append(idx)"
      ],
      "metadata": {
        "id": "mxlyODf-Pg6F"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "n_best = 20\n",
        "max_answer_length = 30\n",
        "predicted_answers = []\n",
        "\n",
        "for example in small_eval_set:\n",
        "    example_id = example[\"id\"]\n",
        "    context = example[\"context\"]\n",
        "    answers = []\n",
        "\n",
        "    for feature_index in example_to_features[example_id]:\n",
        "        start_logit = start_logits[feature_index]\n",
        "        end_logit = end_logits[feature_index]\n",
        "        offsets = eval_set[\"offset_mapping\"][feature_index]\n",
        "\n",
        "        start_indexes = np.argsort(start_logit)[-1 : -n_best - 1 : -1].tolist()\n",
        "        end_indexes = np.argsort(end_logit)[-1 : -n_best - 1 : -1].tolist()\n",
        "        for start_index in start_indexes:\n",
        "            for end_index in end_indexes:\n",
        "                # Skip answers that are not fully in the context\n",
        "                if offsets[start_index] is None or offsets[end_index] is None:\n",
        "                    continue\n",
        "                # Skip answers with a length that is either < 0 or > max_answer_length.\n",
        "                if (\n",
        "                    end_index < start_index\n",
        "                    or end_index - start_index + 1 > max_answer_length\n",
        "                ):\n",
        "                    continue\n",
        "\n",
        "                answers.append(\n",
        "                    {\n",
        "                        \"text\": context[offsets[start_index][0] : offsets[end_index][1]],\n",
        "                        \"logit_score\": start_logit[start_index] + end_logit[end_index],\n",
        "                    }\n",
        "                )\n",
        "\n",
        "    best_answer = max(answers, key=lambda x: x[\"logit_score\"])\n",
        "    predicted_answers.append({\"id\": example_id, \"prediction_text\": best_answer[\"text\"]})"
      ],
      "metadata": {
        "id": "Uka8LrvuPicP"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Fine Tune `BERT` "
      ],
      "metadata": {
        "id": "e6VDDs4wPvih"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = AutoModelForQuestionAnswering.from_pretrained(model_checkpoint)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156,
          "referenced_widgets": [
            "895b055707924d3ca79bf1d967bab680",
            "5f0b43f95bd94e40999be0f63b16b55f",
            "c6fcd278b4214c5bb61e861e28673387",
            "a15c25fca0994671b3fe507bc69b26fd",
            "07781c574be142a4b93e77bb6c711129",
            "49ede41cad4f4cc49b86a5cb7a28af38",
            "89b034aa0ab8414191e549ab032b5187",
            "558e923a8cff41519c2a6bb6c8dc1f8b",
            "a929a3d39b584a7e8a6246f913665bb1",
            "4acdd85e47cf4ef0a2b521ca298bd56d",
            "a7c8b48c662141878f1264e279def104"
          ]
        },
        "id": "vCNgM-tSProT",
        "outputId": "dd85455d-5be7-4c28-ebe8-34c1bb5e2dd7"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading pytorch_model.bin:   0%|          | 0.00/436M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "895b055707924d3ca79bf1d967bab680"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-cased were not used when initializing BertForQuestionAnswering: ['cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.bias']\n",
            "- This IS expected if you are initializing BertForQuestionAnswering from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForQuestionAnswering from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForQuestionAnswering were not initialized from the model checkpoint at bert-base-cased and are newly initialized: ['qa_outputs.bias', 'qa_outputs.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from huggingface_hub import notebook_login\n",
        "\n",
        "notebook_login()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 331,
          "referenced_widgets": [
            "abdd211ddc9a482592d57ead925555bf",
            "d8dfbc85cc7e4654a2378a0a0d077383",
            "a2c42cca09424120a78bb0ae77258e94",
            "962dd21b0b88434ba5a3322cc58766fe",
            "958de1ca3d2849169c69100d5483bb4e",
            "31ef2a1cd9ce42c8a0047c55a1935a9d",
            "fc3a80fc1fd54dd3871eed766d54d629",
            "32ae5bfad33142e7845a8cdf0fb6901b",
            "01a2a6831af949bf933d5f26216a0f8a",
            "be00d7a366394e9dbb0cf9149051aa35",
            "d8a84a0d44164b3cb1ea3c9724a7713a",
            "f3f23aadd37b4f78adc5699faf956d24",
            "df52a0c2d2424eaf9eab3652a8d18510",
            "0bd7b31a5641402c853762881eb95beb",
            "76558554460c48d28c0a27aa9a0acf18",
            "f7ae6d00cc874651991dcc0558afac6b",
            "cfb3246c789f47b4bf0920d03683f087"
          ]
        },
        "id": "21M587PqPyWE",
        "outputId": "306e78a2-2f09-4a10-8da8-8317b0ed9040"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Token is valid.\n",
            "Your token has been saved to /root/.cache/huggingface/token\n",
            "Login successful\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import TrainingArguments\n",
        "\n",
        "args = TrainingArguments(\n",
        "    \"QuAC-QA-BERT\",\n",
        "    evaluation_strategy=\"no\",\n",
        "    save_strategy=\"epoch\",\n",
        "    learning_rate=2e-5,\n",
        "    num_train_epochs=3,\n",
        "    weight_decay=0.01,\n",
        "    fp16=True,\n",
        "    push_to_hub=True,\n",
        ")"
      ],
      "metadata": {
        "id": "nriofekFP2xA"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import Trainer\n",
        "\n",
        "trainer = Trainer(\n",
        "    model=model,\n",
        "    args=args,\n",
        "    train_dataset=train_dataset,\n",
        "    eval_dataset=validation_dataset,\n",
        "    tokenizer=tokenizer,\n",
        ")\n",
        "trainer.train()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 812
        },
        "id": "_p7sztwsP-op",
        "outputId": "87c2fd62-d672-45bc-c972-ed23998a443f"
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Cloning https://huggingface.co/SRDdev/QuAC-QA-BERT into local empty directory.\n",
            "WARNING:huggingface_hub.repository:Cloning https://huggingface.co/SRDdev/QuAC-QA-BERT into local empty directory.\n",
            "/usr/local/lib/python3.9/dist-packages/transformers/optimization.py:391: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n",
            "You're using a BertTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='9513' max='9513' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [9513/9513 40:16, Epoch 3/3]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>2.213000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>1.811800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1500</td>\n",
              "      <td>1.702600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2000</td>\n",
              "      <td>1.656000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2500</td>\n",
              "      <td>1.592600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3000</td>\n",
              "      <td>1.557800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3500</td>\n",
              "      <td>1.354300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4000</td>\n",
              "      <td>1.322400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4500</td>\n",
              "      <td>1.272500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5000</td>\n",
              "      <td>1.327200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5500</td>\n",
              "      <td>1.297700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6000</td>\n",
              "      <td>1.274700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6500</td>\n",
              "      <td>1.211800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7000</td>\n",
              "      <td>0.974800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7500</td>\n",
              "      <td>0.972800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8000</td>\n",
              "      <td>0.958300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8500</td>\n",
              "      <td>0.986500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9000</td>\n",
              "      <td>0.980500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9500</td>\n",
              "      <td>0.965100</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=9513, training_loss=1.337987121642983, metrics={'train_runtime': 2416.4456, 'train_samples_per_second': 31.487, 'train_steps_per_second': 3.937, 'total_flos': 1.4910768774761472e+16, 'train_loss': 1.337987121642983, 'epoch': 3.0})"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "compute_metrics= \"squad\"\n",
        "predictions, _, _ = trainer.predict(validation_dataset)\n",
        "start_logits, end_logits = predictions"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "D-ygQGRyQaU1",
        "outputId": "2ab3c8c7-ed11-400f-f1d7-a562072cbd04"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": []
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.push_to_hub(commit_message=\"Training complete\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 243
        },
        "id": "2T_w0SK7QgQx",
        "outputId": "0073502d-f03b-46b0-a34e-f14efcc0c70a"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "To https://huggingface.co/SRDdev/QuAC-QA-BERT\n",
            "   8dff18b..a1ffe17  main -> main\n",
            "\n",
            "WARNING:huggingface_hub.repository:To https://huggingface.co/SRDdev/QuAC-QA-BERT\n",
            "   8dff18b..a1ffe17  main -> main\n",
            "\n",
            "To https://huggingface.co/SRDdev/QuAC-QA-BERT\n",
            "   a1ffe17..5559501  main -> main\n",
            "\n",
            "WARNING:huggingface_hub.repository:To https://huggingface.co/SRDdev/QuAC-QA-BERT\n",
            "   a1ffe17..5559501  main -> main\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'https://huggingface.co/SRDdev/QuAC-QA-BERT/commit/a1ffe17a71c8dbf5bdb0b41fd7af55e63c3fee47'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Inference 🤗"
      ],
      "metadata": {
        "id": "FKB5dYyKbnpB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer, AutoModelForQuestionAnswering\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"SRDdev/QuAC-QA-BERT\")\n",
        "model = AutoModelForQuestionAnswering.from_pretrained(\"SRDdev/QuAC-QA-BERT\")"
      ],
      "metadata": {
        "id": "XyMMy1afUFOd"
      },
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "context = \"\"\"My name is Sarah and I live in London\"\"\""
      ],
      "metadata": {
        "id": "lAfJQfjLbt6P"
      },
      "execution_count": 50,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import pipeline\n",
        "\n",
        "ask = pipeline(\"question-answering\", model= model , tokenizer = tokenizer)\n",
        "result = ask(question=\"Where do I live?\", context=context)\n",
        "print(f\"Answer: '{result['answer']}'\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PlnHMaYtb1hh",
        "outputId": "a48d7a8c-74fb-43d6-bf71-7c99ed59216b"
      },
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Answer: 'My name is Sarah and I live in London'\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "LERzMZgGb6jF"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}