BERT_BASE_NER_Notebook.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BERT-BASE-NER_Notebook.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "17014cdd564943428c4ecc8a95ee8bf0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4d3a5f97ce684cf8b47f9535b3c7e9a5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_83816fecdacf409481210b1879f1e717",
              "IPY_MODEL_ae9d02c538ae4bcf94abbcb6927d2d3b",
              "IPY_MODEL_e17db36b7c13468bbc80562c1b45b651"
            ]
          }
        },
        "4d3a5f97ce684cf8b47f9535b3c7e9a5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "83816fecdacf409481210b1879f1e717": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c2f92fda33a9492f82cfbb588fb4f643",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_12c8faedd714422ca704c3ef9c4c2657"
          }
        },
        "ae9d02c538ae4bcf94abbcb6927d2d3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d0ddae70e3c04c00a06c2f714884db68",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 59,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 59,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ad8a94ee8686433ab57e72554b2c5235"
          }
        },
        "e17db36b7c13468bbc80562c1b45b651": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_94f18f957f2e4cddbe46782b52928128",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 59.0/59.0 [00:00&lt;00:00, 1.02kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f8ccb648bc504fa29a759f3c31788a94"
          }
        },
        "c2f92fda33a9492f82cfbb588fb4f643": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "12c8faedd714422ca704c3ef9c4c2657": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d0ddae70e3c04c00a06c2f714884db68": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ad8a94ee8686433ab57e72554b2c5235": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "94f18f957f2e4cddbe46782b52928128": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f8ccb648bc504fa29a759f3c31788a94": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "59112a588fc941c394a39b5d31fec22c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d92980aeb2b84bdbaddff4fbab4a0fdc",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_47a9f3aa059a4569b4980830099fdbd5",
              "IPY_MODEL_017b8020fffb46d4a385ed04c8f09d9c",
              "IPY_MODEL_77b70aec88e2492a999eb3f6951dedc5"
            ]
          }
        },
        "d92980aeb2b84bdbaddff4fbab4a0fdc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "47a9f3aa059a4569b4980830099fdbd5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c9b384ae399441f6bf3df33ea484df67",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a720d83170e54a2389d594eedba89dc9"
          }
        },
        "017b8020fffb46d4a385ed04c8f09d9c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_af4eb22ce89c467e9318f5d7e0e78207",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 829,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 829,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bc800be3d9904a8fa695ebe6a5fe87a5"
          }
        },
        "77b70aec88e2492a999eb3f6951dedc5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fd12607393364ac79027ce90d067a997",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 829/829 [00:00&lt;00:00, 22.0kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ade30dba960e44fa8ad5e57e467a0d39"
          }
        },
        "c9b384ae399441f6bf3df33ea484df67": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a720d83170e54a2389d594eedba89dc9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "af4eb22ce89c467e9318f5d7e0e78207": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bc800be3d9904a8fa695ebe6a5fe87a5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fd12607393364ac79027ce90d067a997": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ade30dba960e44fa8ad5e57e467a0d39": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "68336da936574bda86d1aba7bebb6da7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_06e7bbdf112842abb3acbdaceead379c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_215d3287fcba4490a9444b53aad16330",
              "IPY_MODEL_4c5c3c66006d451da33974f5d3dc1793",
              "IPY_MODEL_c700b4245c704520aeb827e22f850218"
            ]
          }
        },
        "06e7bbdf112842abb3acbdaceead379c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "215d3287fcba4490a9444b53aad16330": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_72fc18fe523a4d059e469d127e63172d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e56a8aedb1ae4398bb2f22fe8cc29f9e"
          }
        },
        "4c5c3c66006d451da33974f5d3dc1793": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6fe240c1097c4006bf5b4c6bf2a2064b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 213450,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 213450,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_abc854ae81fa4d618ce702574de03e5b"
          }
        },
        "c700b4245c704520aeb827e22f850218": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1224212040974582af4c906b689146df",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 208k/208k [00:00&lt;00:00, 1.14MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_eefdea62b3c24a0f9d024d5dbe621f00"
          }
        },
        "72fc18fe523a4d059e469d127e63172d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e56a8aedb1ae4398bb2f22fe8cc29f9e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6fe240c1097c4006bf5b4c6bf2a2064b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "abc854ae81fa4d618ce702574de03e5b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1224212040974582af4c906b689146df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "eefdea62b3c24a0f9d024d5dbe621f00": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5a9a21db05f1407ca1be110b9a48c6b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f3dd2bba7bf945b6a06f5e1570bdc418",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4d1542553dbd4684b96c56440aa31f8d",
              "IPY_MODEL_25af399dab7e4eb2878c927bf76df02f",
              "IPY_MODEL_e33bfb2476b549819994f881461103bb"
            ]
          }
        },
        "f3dd2bba7bf945b6a06f5e1570bdc418": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4d1542553dbd4684b96c56440aa31f8d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_48c767a317cc469481752c4c75f6ac44",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_caf0ed8833e44e46859c279300fa20b0"
          }
        },
        "25af399dab7e4eb2878c927bf76df02f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_706a70ec2f48437e909c0f8032b74e10",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cfb93e98cd4047738f98d702e96966bd"
          }
        },
        "e33bfb2476b549819994f881461103bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7ebd8dec1aa64026bad915a6a29ca14d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2.00/2.00 [00:00&lt;00:00, 48.8B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_26b2de5989194446beec03768a9d2e2d"
          }
        },
        "48c767a317cc469481752c4c75f6ac44": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "caf0ed8833e44e46859c279300fa20b0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "706a70ec2f48437e909c0f8032b74e10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cfb93e98cd4047738f98d702e96966bd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7ebd8dec1aa64026bad915a6a29ca14d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "26b2de5989194446beec03768a9d2e2d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bc2135da26524a338e66950293f2ff5c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_485ff5500d5446259a1f7ab04fa62ead",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_cdbb50d2c0414d848f7a6a401158cfb9",
              "IPY_MODEL_0a21252a73cf425f8194e28766e04d18",
              "IPY_MODEL_58bd6615d5574f1f938ef796027a3def"
            ]
          }
        },
        "485ff5500d5446259a1f7ab04fa62ead": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cdbb50d2c0414d848f7a6a401158cfb9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ae887e71e40e4b508e97c62aebd1c02b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6fe009b584854db484d4533eae9a759a"
          }
        },
        "0a21252a73cf425f8194e28766e04d18": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7c58fdc970674b6a919f71dfaa448170",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 112,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 112,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_63ad289c0132459f9d9f04c6994a6d95"
          }
        },
        "58bd6615d5574f1f938ef796027a3def": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ead452efd31c48b39f7d251de1f10b5e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 112/112 [00:00&lt;00:00, 2.63kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_35f80ca3ce7948f4b1d02d65ea86d8e1"
          }
        },
        "ae887e71e40e4b508e97c62aebd1c02b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6fe009b584854db484d4533eae9a759a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7c58fdc970674b6a919f71dfaa448170": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "63ad289c0132459f9d9f04c6994a6d95": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ead452efd31c48b39f7d251de1f10b5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "35f80ca3ce7948f4b1d02d65ea86d8e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6078eebc446847edab07b950c4d34bbc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3dee0176d87e439caa95e1e111077798",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_180b1e0fa1774aa3bfe6a3485cf8fbee",
              "IPY_MODEL_987de82fc43c43609d15f07c34350946",
              "IPY_MODEL_4bda05a6b9f84c94a6f1cb42dc99f71c"
            ]
          }
        },
        "3dee0176d87e439caa95e1e111077798": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "180b1e0fa1774aa3bfe6a3485cf8fbee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_266704ea773e41638d9b906163ad72e1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bcfa196f0374464597fda9af0eb854cc"
          }
        },
        "987de82fc43c43609d15f07c34350946": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9cfa865750e64e10a57f64ba0b5e842f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 433316646,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 433316646,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4fb57cc63fa54646a46c76c7bf96a0c0"
          }
        },
        "4bda05a6b9f84c94a6f1cb42dc99f71c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c680b526a4304283b7d0a70cc4cb3e49",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 413M/413M [00:15&lt;00:00, 29.8MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_296f6bc713264c3d8d0462514f8cb998"
          }
        },
        "266704ea773e41638d9b906163ad72e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bcfa196f0374464597fda9af0eb854cc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9cfa865750e64e10a57f64ba0b5e842f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4fb57cc63fa54646a46c76c7bf96a0c0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c680b526a4304283b7d0a70cc4cb3e49": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "296f6bc713264c3d8d0462514f8cb998": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IswjtajVf-YO"
      },
      "source": [
        "## Setup\n",
        "\n",
        "The set up phase of this notebook begins with mounting your google drive by running the first code chunk, the output will ask you to input an access token that can be found by clicking the provided link and signing into the google account associated with the current Colab session. This step allows you to save the doccano output somewhere in your drive without having to reupload the document every time you begin a new session (or the kernal restarts).\n",
        "\n",
        "Once the drive is mounted, run the following code chunks IN THE ORDER THEY ARE PROVIDED. Failing to do so may cause errors do to potential conflicts between the `torch` and `transformer` libraries."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CUMXsZ7_caLi",
        "outputId": "fcb47ef7-8bc6-442c-ff6d-402a16eeeade"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LK_ud_NAka_4"
      },
      "source": [
        "#!curl -q https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n",
        "#!python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "JYeja6oJECbT",
        "outputId": "c1564646-c77c-4f37-ff1c-7196fa247366"
      },
      "source": [
        "import torch\n",
        "torch.__version__"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'1.10.0+cu111'"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lbxud-ymEJUN",
        "outputId": "7dfbaf63-8d9b-4257-cb71-7931dacda9fa"
      },
      "source": [
        "import torch.autograd as autograd\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import json\n",
        "from collections import defaultdict, OrderedDict\n",
        "import math\n",
        "import numpy as np\n",
        "import random\n",
        "\n",
        "torch.manual_seed(1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<torch._C.Generator at 0x7f20776254b0>"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ro4_Bu7OhWsF",
        "outputId": "9a477ad4-abd0-411d-d8c6-14e41a6c72df"
      },
      "source": [
        "!pip install transformers\n",
        "!pip -q install seqeval"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.12.5)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.46)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.3)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.2.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.4.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.62.3)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.8.2)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.6)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.6.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.10.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.1.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vHzHqMC6xZPt"
      },
      "source": [
        "from seqeval.metrics import classification_report as classification_report_seqeval\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from transformers import AutoTokenizer, AutoModelForTokenClassification\n",
        "from transformers import pipeline"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G8_3MXovhaL4"
      },
      "source": [
        "## Initialize GPU"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AYfoSkauEOWE",
        "outputId": "6da73902-fc82-448b-ab98-7da5c704f842"
      },
      "source": [
        "dev = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "dev"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "device(type='cuda', index=0)"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3UWhZubSK2FO"
      },
      "source": [
        "## Load data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kj3ebBppKy6H",
        "outputId": "d769c9dd-b863-4ca8-8271-6854a2c84038"
      },
      "source": [
        "# Read export file\n",
        "## CHANGE THIS DIRECTORY TO WHERE THE DOCCANO TEST IS\n",
        "with open('/content/drive/MyDrive/ML_Models/Bi-LSTM/doccano_output.jsonl', 'r') as f:\n",
        "    lines = f.readlines()\n",
        "\n",
        "len(lines)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "108"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iYLP82KbfM_E"
      },
      "source": [
        "def remap(x: str):\n",
        "    '''\n",
        "    function for remapping labels from B-POS and I-POS to B-MISC and I-MISC\n",
        "    '''\n",
        "    if x == 'B-POS':\n",
        "        return 'B-MISC'\n",
        "    elif x == 'I-POS':\n",
        "        return 'I-MISC'\n",
        "    else:\n",
        "        return x"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BpsEEPnPlRaW"
      },
      "source": [
        "# The numerical doccano label to actual label (B-I-O scheme)\n",
        "\n",
        "ix_to_label = {0: 'B-PER', 1: 'I-PER', 2:'B-MISC', 3:'I-MISC', 4:'B-ORG', 5: 'I-ORG', 6: 'O'}\n",
        "    \n",
        "# train/test data\n",
        "data = []\n",
        "\n",
        "# Vocabulary\n",
        "vocab = set()\n",
        "    \n",
        "# Loop over each data point (a corpus of labeled text) to extract words\n",
        "for line in lines:\n",
        "    \n",
        "    # An ordered dict will keep items in order for further manipulation\n",
        "    # so we initialize here\n",
        "    orddict = OrderedDict({})\n",
        "\n",
        "    # Lists to hold the words and labels\n",
        "    words = []\n",
        "    labels = []\n",
        "    # Convert line to json\n",
        "    input_json = json.loads(line)\n",
        "    annots = [[x[0], x[1], remap(x[2])] for x in input_json['label']] # call remap function to change token values for BERT-BASE-NER classification\n",
        "    text = input_json['data']\n",
        "\n",
        "    # Add each word annotation to OrderedDict\n",
        "    for ann in annots:\n",
        "        orddict[ann[0]] = ann\n",
        "    \n",
        "    # Sort ordered dict because there's no guarantee reading json\n",
        "    # maintained order\n",
        "    orddict = sorted(orddict.items(), key=lambda x: x[1][0])\n",
        "    \n",
        "    for item in orddict:\n",
        "        # the item is a tuple where second value is the actual value we want\n",
        "        ann = item[1]\n",
        "        # Subset text string\n",
        "        word = text[ann[0]:(ann[1])].rstrip()\n",
        "        label = ann[2]\n",
        "        # Add to list for this datum/corpus\n",
        "        words.append(word)\n",
        "        labels.append(label)\n",
        "        vocab.add(word)\n",
        "    # Add to overall data containers\n",
        "    data.append((words, labels))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dTvF4Uq7lUwK"
      },
      "source": [
        "ix_to_label =  {0: 'B-PER', 1: 'I-PER', 2:'B-MISC', 3:'I-MISC', 4:'B-ORG', 5: 'I-ORG', 6: 'O'}\n",
        "tags_vals = [v for k,v in ix_to_label.items()]\n",
        "tag2idx = {t: i for i, t in ix_to_label.items()}\n",
        "sentences = [' '.join([s for s in sent[0]]) for sent in data]\n",
        "labels = [[s for s in sent[1]] for sent in data]\n",
        "labels = [[tag2idx.get(l) for l in lab] for lab in labels]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N56IBfJej2Lr"
      },
      "source": [
        "### BERT-BASE-NER"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v1p6rkMdj9s3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 209,
          "referenced_widgets": [
            "17014cdd564943428c4ecc8a95ee8bf0",
            "4d3a5f97ce684cf8b47f9535b3c7e9a5",
            "83816fecdacf409481210b1879f1e717",
            "ae9d02c538ae4bcf94abbcb6927d2d3b",
            "e17db36b7c13468bbc80562c1b45b651",
            "c2f92fda33a9492f82cfbb588fb4f643",
            "12c8faedd714422ca704c3ef9c4c2657",
            "d0ddae70e3c04c00a06c2f714884db68",
            "ad8a94ee8686433ab57e72554b2c5235",
            "94f18f957f2e4cddbe46782b52928128",
            "f8ccb648bc504fa29a759f3c31788a94",
            "59112a588fc941c394a39b5d31fec22c",
            "d92980aeb2b84bdbaddff4fbab4a0fdc",
            "47a9f3aa059a4569b4980830099fdbd5",
            "017b8020fffb46d4a385ed04c8f09d9c",
            "77b70aec88e2492a999eb3f6951dedc5",
            "c9b384ae399441f6bf3df33ea484df67",
            "a720d83170e54a2389d594eedba89dc9",
            "af4eb22ce89c467e9318f5d7e0e78207",
            "bc800be3d9904a8fa695ebe6a5fe87a5",
            "fd12607393364ac79027ce90d067a997",
            "ade30dba960e44fa8ad5e57e467a0d39",
            "68336da936574bda86d1aba7bebb6da7",
            "06e7bbdf112842abb3acbdaceead379c",
            "215d3287fcba4490a9444b53aad16330",
            "4c5c3c66006d451da33974f5d3dc1793",
            "c700b4245c704520aeb827e22f850218",
            "72fc18fe523a4d059e469d127e63172d",
            "e56a8aedb1ae4398bb2f22fe8cc29f9e",
            "6fe240c1097c4006bf5b4c6bf2a2064b",
            "abc854ae81fa4d618ce702574de03e5b",
            "1224212040974582af4c906b689146df",
            "eefdea62b3c24a0f9d024d5dbe621f00",
            "5a9a21db05f1407ca1be110b9a48c6b7",
            "f3dd2bba7bf945b6a06f5e1570bdc418",
            "4d1542553dbd4684b96c56440aa31f8d",
            "25af399dab7e4eb2878c927bf76df02f",
            "e33bfb2476b549819994f881461103bb",
            "48c767a317cc469481752c4c75f6ac44",
            "caf0ed8833e44e46859c279300fa20b0",
            "706a70ec2f48437e909c0f8032b74e10",
            "cfb93e98cd4047738f98d702e96966bd",
            "7ebd8dec1aa64026bad915a6a29ca14d",
            "26b2de5989194446beec03768a9d2e2d",
            "bc2135da26524a338e66950293f2ff5c",
            "485ff5500d5446259a1f7ab04fa62ead",
            "cdbb50d2c0414d848f7a6a401158cfb9",
            "0a21252a73cf425f8194e28766e04d18",
            "58bd6615d5574f1f938ef796027a3def",
            "ae887e71e40e4b508e97c62aebd1c02b",
            "6fe009b584854db484d4533eae9a759a",
            "7c58fdc970674b6a919f71dfaa448170",
            "63ad289c0132459f9d9f04c6994a6d95",
            "ead452efd31c48b39f7d251de1f10b5e",
            "35f80ca3ce7948f4b1d02d65ea86d8e1",
            "6078eebc446847edab07b950c4d34bbc",
            "3dee0176d87e439caa95e1e111077798",
            "180b1e0fa1774aa3bfe6a3485cf8fbee",
            "987de82fc43c43609d15f07c34350946",
            "4bda05a6b9f84c94a6f1cb42dc99f71c",
            "266704ea773e41638d9b906163ad72e1",
            "bcfa196f0374464597fda9af0eb854cc",
            "9cfa865750e64e10a57f64ba0b5e842f",
            "4fb57cc63fa54646a46c76c7bf96a0c0",
            "c680b526a4304283b7d0a70cc4cb3e49",
            "296f6bc713264c3d8d0462514f8cb998"
          ]
        },
        "outputId": "2bf35b1e-278a-4ff6-bf9a-806c36afa3c1"
      },
      "source": [
        "# Defining tokenizer and model\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"dslim/bert-base-NER\")\n",
        "model = AutoModelForTokenClassification.from_pretrained(\"dslim/bert-base-NER\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "17014cdd564943428c4ecc8a95ee8bf0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/59.0 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "59112a588fc941c394a39b5d31fec22c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/829 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "68336da936574bda86d1aba7bebb6da7",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/208k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5a9a21db05f1407ca1be110b9a48c6b7",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/2.00 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bc2135da26524a338e66950293f2ff5c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/112 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6078eebc446847edab07b950c4d34bbc",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/413M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dZJQPNQOiNqa"
      },
      "source": [
        "def label_entities(sequences):\n",
        "    tokens_list = []\n",
        "    labels_list = []\n",
        "    for s in sequences:\n",
        "        inputs = tokenizer(s, return_tensors = 'pt')\n",
        "        tokens = inputs.tokens()\n",
        "\n",
        "        outputs = model(**inputs).logits\n",
        "        predictions = torch.argmax(outputs, dim=2)\n",
        "        tokens_list.append(tokens)\n",
        "        labels_list.extend([model.config.id2label[x] for x in predictions[0].numpy()])\n",
        "    return tokens_list, labels_list\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bUcONCeKlW8G",
        "outputId": "b03c9b07-44bc-495d-96a9-60f78a1d68ef"
      },
      "source": [
        "# Pull lists of predicted tokens\n",
        "tokens, pred_labels = label_entities(sentences)\n",
        "\n",
        "# Map original tokens back to tag space\n",
        "flat_labels = [tags_vals[l_i] if l_i is not None else 'O'  for l in labels for l_i in l]\n",
        "pd.Series(flat_labels).value_counts()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "O         2718\n",
              "I-ORG      346\n",
              "I-MISC     162\n",
              "B-ORG      156\n",
              "I-PER      120\n",
              "B-PER      109\n",
              "B-MISC      89\n",
              "dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HdgepFl2xMQ3"
      },
      "source": [
        "### Comparing custom labels with new labels\n",
        "\n",
        "Unfortunately, due to the sub-word tokenization techniques used in the BERT-BASE-NER model the number of tokens, and consequent number of labels, are going to be different. Therefore,, we cannot use the same `classification_report_seqeval` that we used in the BiLSTM-CRF and the fine-tuned bert-base-* models. An example of the differences between sequences can be seen in the second code chunk below."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 362
        },
        "id": "McWoWlaymGac",
        "outputId": "401db169-765a-4ecd-c360-54e399df1a7a"
      },
      "source": [
        "classification_report_seqeval([flat_labels], [pred_labels])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-15-b559e2d85bd7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mclassification_report_seqeval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mflat_labels\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mpred_labels\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/seqeval/metrics/sequence_labeling.py\u001b[0m in \u001b[0;36mclassification_report\u001b[0;34m(y_true, y_pred, digits, suffix, output_dict, mode, sample_weight, zero_division, scheme)\u001b[0m\n\u001b[1;32m    695\u001b[0m         \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    696\u001b[0m         \u001b[0mzero_division\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mzero_division\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 697\u001b[0;31m         \u001b[0msuffix\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msuffix\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    698\u001b[0m     )\n\u001b[1;32m    699\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mrow\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarget_names\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/seqeval/metrics/sequence_labeling.py\u001b[0m in \u001b[0;36mprecision_recall_fscore_support\u001b[0;34m(y_true, y_pred, average, warn_for, beta, sample_weight, zero_division, suffix)\u001b[0m\n\u001b[1;32m    137\u001b[0m         \u001b[0mscheme\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    138\u001b[0m         \u001b[0msuffix\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msuffix\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 139\u001b[0;31m         \u001b[0mextract_tp_actual_correct\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mextract_tp_actual_correct\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    140\u001b[0m     )\n\u001b[1;32m    141\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/seqeval/metrics/v1.py\u001b[0m in \u001b[0;36m_precision_recall_fscore_support\u001b[0;34m(y_true, y_pred, average, warn_for, beta, sample_weight, zero_division, scheme, suffix, extract_tp_actual_correct)\u001b[0m\n\u001b[1;32m    120\u001b[0m         \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'average has to be one of {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maverage_options\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    121\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 122\u001b[0;31m     \u001b[0mcheck_consistent_length\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_true\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_pred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    123\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    124\u001b[0m     \u001b[0mpred_sum\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtp_sum\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrue_sum\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mextract_tp_actual_correct\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_true\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msuffix\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscheme\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/seqeval/metrics/v1.py\u001b[0m in \u001b[0;36mcheck_consistent_length\u001b[0;34m(y_true, y_pred)\u001b[0m\n\u001b[1;32m     99\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_true\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_pred\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mlen_true\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mlen_pred\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m         \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'Found input variables with inconsistent numbers of samples:\\n{}\\n{}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen_true\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen_pred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 101\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: Found input variables with inconsistent numbers of samples:\n[3700]\n[4913]"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ji2J4onK56An"
      },
      "source": [
        "Part of the reason for this difference in sample sizes is likely that the original data does not include the `[CLS]` and `[SEP]` identifiers that indicate the beginning and end of each sentence."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ICB8fNTDxdv3",
        "outputId": "472f03fd-219c-4a6a-b379-101f2d147f4f"
      },
      "source": [
        "print(data[1][0])\n",
        "print(tokens[1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['In', 'the', 'wake', 'of', 'nationwide', 'protests', 'about', 'racial', 'injustice', ',', 'we', 'partnered', 'with', 'CNN', 'to', 'air', 'a', 'town', 'hall', ',', 'Coming', 'Together', ':', 'Standing', 'Up', 'to', 'Racism', ',', 'and', 'launched', 'a', 'special', ',', 'The', 'Power', 'of', 'We', ',', 'on', 'PBS', 'Kids', 'and', 'HBO', 'Max']\n",
            "['[CLS]', 'In', 'the', 'wake', 'of', 'nationwide', 'protests', 'about', 'racial', 'injustice', ',', 'we', 'partnered', 'with', 'CNN', 'to', 'air', 'a', 'town', 'hall', ',', 'Coming', 'Together', ':', 'Standing', 'Up', 'to', 'Ra', '##cis', '##m', ',', 'and', 'launched', 'a', 'special', ',', 'The', 'Power', 'of', 'We', ',', 'on', 'PBS', 'Kids', 'and', 'HBO', 'Max', '[SEP]']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z4trvmlIwiNw"
      },
      "source": [
        "### An alternative method of evaluation\n",
        "\n",
        "Since the use of the NER pipeline includes a sub-word tokenizer that maps to a different number of outputs for each sequence, we are instead going to evaluate the distribution of entity types for both the handlabled tags and the tags returned by the huggingface pipeline.\n",
        "\n",
        "Although this will serve to somewhat show differneces between the original tags and those assigned by the HuggingFace pipeline, there are a few remaining issues with this comparison. The primary issue is incomensurate tag-space, wherein the bert-base-ner incorporates an additional `'LOC'` set of tags indicating locations. Additionally, the reduction of the position category `'POS'` into a `'MISC'` category is not ideal, as the NER model may not regard the position of a person as a miscellaneous entity.\n",
        "\n",
        "Nevertheless, we will look at a few key differences in the plots below to identify some advantages and disadvantages of the huggingface pipeline approach.\n",
        "\n",
        "#### Distribution of Tags\n",
        "\n",
        "The plot and value counts below demonstrate a similar *relative* prevalence of `'O'` tags to the others but, as expected with the differences in tokenization methods, the pipeline includes far more tag counts across all of the categories due to the use of sub-tokens."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 692
        },
        "id": "JqOwHVo_jvTI",
        "outputId": "6f6aff9e-c797-4f6f-cebc-0b48c5cd761d"
      },
      "source": [
        "fig, axes = plt.subplots(nrows=1, ncols=2)\n",
        "\n",
        "original_s = pd.Series(flat_labels)\n",
        "predicted_s = pd.Series(pred_labels)\n",
        "\n",
        "original_s.value_counts().sort_index().plot(kind = 'bar',title = 'Original', ax=axes[0])\n",
        "predicted_s.value_counts().sort_index().plot(kind = 'bar',title = 'HuggingFace', ax=axes[1])\n",
        "\n",
        "print(\"Original Tag Counts: \\n\", original_s.value_counts().sort_index(),  \"\\n\\nPipeline Tag Counts: \\n\", predicted_s.value_counts().sort_index())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original Tag Counts: \n",
            " B-MISC      89\n",
            "B-ORG      156\n",
            "B-PER      109\n",
            "I-MISC     162\n",
            "I-ORG      346\n",
            "I-PER      120\n",
            "O         2718\n",
            "dtype: int64 \n",
            "\n",
            "Pipeline Tag Counts: \n",
            " B-LOC       19\n",
            "B-MISC      39\n",
            "B-ORG      197\n",
            "B-PER      126\n",
            "I-LOC       36\n",
            "I-MISC      72\n",
            "I-ORG      687\n",
            "I-PER      166\n",
            "O         3571\n",
            "dtype: int64\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEhCAYAAACTNXDdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dfbxdVX3n8c/XBCIKCJgUIzAGMYjYQqApxJahVioEWic4Y32BVR4tOENanTptgT6AD3TQClRH5FWQKLaWlEGoUdNiig4dGHkIFJAEeBF5aKDhMTxEEW3gO3/sdeXk5j6ce84+995z9vf9ep3XPWftvdfZZ+11fneftdfaS7aJiIhmeMVU70BEREyeBP2IiAZJ0I+IaJAE/YiIBknQj4hokAT9iIgGSdDvI5LOlPTFutdtIy9LelMdeUWMRdIPJb1xqvdjkCXoTyFJJ0j6vqTnJT0q6SJJO422vu0/t/3BdvKeyLoRrSQ9KOnXh6WdIOn6Xr+37e1t399tPmV/Xyz/RIYen69jH/tdgv4UkfRR4FPAHwCvARYBbwBWSdp2hPVnTu4eRvS975V/IkOPpVO9Q9NBgv4UkLQj8DHgd23/o+1/t/0g8F5gHvB+SWdLulLS30h6DjihpP1NSz7HSXpI0lOS/rT1DK11XUnzShPN8ZL+VdKTkv64JZ+DJH1P0jOSNkj6/Ej/eCJg6+Y+SV+W9MmW139Y6tG/Sfpg6/qSXivpG5Kek3SLpE+2/oIYtu6XJV0o6VuSNkm6SdJeLeseLuleSc9K+oKk6ySN+etW0omS7i753S/p1GHLl0i6vezfDyQtLumvkXRp+VyPlP2e0W1ZToUE/anxy8ArgataE23/EFgJvLMkLQGuBHYCvtq6rqR9gS8Avw3Mpfq1sNs473sI8GbgMODPJL2lpL8I/HdgNvC2svy/dfC5ouFKkPx94NeBNwFvH7bKhcCPgNcBx5fHWI6hOkHaGVgHnFPeZzbVd+MM4LXAvVTfq/E8DvwmsCNwInCBpANLngcBX6H69b0TcCjwYNnuy8Dm8pkOAA4H+rL5NEF/aswGnrS9eYRlG8pyqH6e/r3tl2z/eNh67wG+Yft62z8F/gwY70ZKH7P9Y9t3AHcA+wPYvtX2jbY3l18cfwX8amcfLQbE35dffs9IeobqBKMd7wW+ZHuN7eeBs4cWlDPj/wKcZft522uBy8bJ72rbN5fvyleBBSX9KGCN7avKss8Bjw7bdlHrZ5C0yPa3bP/AleuAbwP/sax/MrDM9qrynXvE9j2Sdi3v9xHbP7L9OHAB1T+kvpN24qnxJDBb0swRAv/cshxg/Rh5vL51ue3nJT01zvu2fimeB7YHkLQ3cD6wEHgVVb24dbwPEQPtaNv/NPRC0gm0d2b7emB1y+vWOjyHqm6tH2X5SEass2xd/y3p4WHb3mj7kNYESUcCZwF7U530vgr4flm8B9Uv7eHeAGwDbJA0lPaKNvZ9WsqZ/tT4HvAT4D+3JkraHjgSuLYkjXXmvgHYvWXb7ah+5nbiIuAeYL7tHYEzAY29STTY81TBcsjrWp5vUS+pAumQJ6iaSEZbPhHD67+G5bsVSbOArwGfAXa1vRNVkB+q6+uBvUbYdD3V93W27Z3KY0fbb+1w36dUgv4UsP0sVTvl/5K0WNI2kuYBVwAPA3/dRjZXAu+S9MvlouvZdB6odwCeA34oaR/gv3aYTzTD7cD7JM0obfitTYFXACdKeoukVwF/OrTA9otU17HOlvSqUteO63AfvgX8gqSjS8+209jyn89ItgVmUf75lLP+w1uWX1r2/TBJr5C0m6R9bG+gagY6T9KOZdlekvqyCTRBf4rY/jTVGfVnqALuTVRnFIfZ/kkb268BfhdYTnXW80Oqi1TjbjuC/wG8D9gEXAL8XQd5RHN8GHgX8AxVR4K/H1pg+x+o2te/S3Xh9cayaKheLqXqdPAo1cnN5XRQZ20/CfwW8GngKWBfqmalUfOyvQn4Pap/TE9T1fkVLctvplzcBZ4FrqNq2oHqn9O2wNqy7ZVUTbF9R5lEZTCUpqFnqJpoHpjq/YkAKD3E7gJmjdRxQdKngNfZHq8Xz3jv8wqqX8m/bfu73eQ16HKm38ckvav8TH411S+G7/NyF7OIKSHp3ZJmSdqZagDiN4YCvqR9JO2nykFUPWau7vB9jpC0U2mrH7oOdeM4mzVegn5/WwL8W3nMB45xfrrF1DuVqqnxB1RjQFqvEe1A1a7/I6pmxPOAr3f4Pm8r7/EkVXPT0SN0bY5h0rwTEdEgOdOPiGiQaT04a/bs2Z43b95U70YMsFtvvfVJ23Mm+31Tt6OXxqrX0zroz5s3j9WrV4+/YkSHJD00Fe+buh29NFa9TvNORESDJOhHRDRIgn5ERIMk6EdENEiCfkREgyToR0Q0SIJ+RESDJOhHRDRIgn5ERINM6xG5EWOZd/q32l73wXN/o4d7EtFb49X1idTvnOlHY73wwgsAb5F0h6Q1kj4GIOnLkh6QdHt5LCjpkvQ5Sesk3SnpwKG8JB0v6b7y6GpCkIheypl+NNasWbMA7rV9oKRtgOsl/UNZ/Ae2rxy2yZFU8xbMBw6mmlD+YEm7AGcBC6kms79V0grbT0/G54iYiJzpR2NJAnipvNymPMaaYGIJ8BVXbgR2kjQXOAJYZXtjCfSrgMW92/OIziXoR+NJup1qpqdVtm8qyeeUJpwLynR8ALtRTV4/5OGSNlr68Pc5RdJqSaufeOKJ2j9HRDsS9KPxbC8AdgcOkvTzwBnAPsAvAbsAf1TT+1xse6HthXPmTPot/COABP0IAGw/A3wXWGx7Q2nC+QnwJeCgstojwB4tm+1e0kZLj5h2EvSjsUoTywwASdsB7wTuKe30qGr0Pxq4q2yyAjiu9OJZBDxrewNwDXC4pJ0l7QwcXtIipp303onG2rBhA8CbJd1JdQJ0he1vSvqOpDmAgNuBD5VNVgJHAeuA54ETAWxvlPQJ4Jay3sdtb5y8TxLRvgT9aKz99tsPYK3tha3ptt8x0vq2DZw2yrJlwLK69zGibmneiYhokAT9iIgGSdCPiGiQcYO+pD0kfVfS2nJ/kg+X9LMlPdJyf5KjWrY5o9yf5F5JR7SkLy5p6ySd3puPFBERo2nnQu5m4KO2b5O0A9V9RVaVZRfY/kzrypL2BY4B3gq8HvgnSXuXxRdSdYt7GLil3J9kbR0fJCIixjdu0C/9kDeU55sk3c0IQ8xbLAGWl4EtD0hax8uDW9bZvh9A0vKyboJ+RMQkmVCbvqR5wAHA0P1Jlpb7kywrg1Ig9yeJiJi22g76krYHvgZ8xPZzVLeV3QtYQPVL4Lw6dij3J4mI6J22BmeVe41/Dfiq7asAbD/WsvwS4Jvl5Vj3Icn9SSIiplA7vXcEXArcbfv8lvS5Lau9my3vT3KMpFmS9qSacOJmqiHq8yXtKWlbqou9K+r5GBER0Y52zvR/BfgA8P1y33GAM4FjyzRyBh4ETgWwvUbSFVQXaDcDp9l+EUDSUqobUc0AltleU+NniYiIcbTTe+d6qhtPDbdyjG3OAc4ZIX3lWNtFRERvZURuRESDJOhHRDRIgn5ERIMk6EdENEiCfkREgyToR0Q0SIJ+RESDJOhHRDRIgn401gsvvADwFkl3lAmCPgZQbhVyU5ns5+/KbUMotxb5u5J+U7nrLGXZiBMHRUw3CfrRWLNmzQK41/b+VHeLXSxpEfApqgmC3gQ8DZxcNjkZeLqkX1DWGz5x0GLgC5JmTOZniWhXgn40VnUvQV4qL7cpDwPvAK4s6ZcBR5fnS8pryvLDyg0JfzZxkO0HgNaJgyKmlQT9aLxyI8HHgVXAD4BnbG8ui1sn+/nZREBl+bPAa8kEQdFHEvSj8WwvoJrf4SBgnx6+TyYIiimXoB8B2H4G+C7wNmAnSUN3oG2d7OdnEwSV5a8BnmLsiYMippUE/Wis0sQyA0DSdsA7gbupgv97ymrHA18vz1eU15Tl37FtRp84KGLaaWu6xIhBtGHDBoA3S7qT6gToCtvflLQWWC7pk8C/UM0cR/n715LWARupeuyMOXFQxHSToB+Ntd9++wGstb2wNd32/YzQ+8b2C8BvjZTXaBMHRUw3ad6JiGiQBP2IiAZJ0I+IaJAE/YiIBknQj4hokAT9iIgGSdCPiGiQBP2IiAZJ0I+IaJAE/YiIBknQj4hokAT9iIgGGTfoS9pD0nclrS2TR3+4pO8iaZWk+8rfnUu6JH2uTBJ9p6QDW/I6vqx/n6TjR3vPiIjojXbO9DcDH7W9L7AIOK1MBH06cK3t+cC15TXAkVT3E58PnAJcBNU/CeAs4GCqOxieNfSPIiIiJse4Qd/2Btu3leebqCaZ2I0tJ4kePnn0V1y5kWoWornAEcAq2xttP001H+niWj9NRESMaUJt+pLmAQcANwG72t5QFj0K7FqejzZJdCaPjoiYYm0HfUnbA18DPmL7udZlZco417FDmTw6IqJ32gr6krahCvhftX1VSX6sNNtQ/j5e0kebJDqTR0dETLF2eu+Iam7Qu22f37KodZLo4ZNHH1d68SwCni3NQNcAh0vauVzAPbykRUTEJGlnjtxfAT4AfF/S7SXtTOBc4ApJJwMPAe8ty1YCRwHrgOeBEwFsb5T0CeCWst7HbW+s5VNERERbxg36tq8HNMriw0ZY38Bpo+S1DFg2kR2M6JX169cD7C1pLdU1qYttf1bS2cDvAEM9Cc60vRJA0hnAycCLwO/ZvqakLwY+C8wAvmj73Mn8LBHtaudMP2IgzZw5E+Bh2/tK2gG4VdKqsvgC259pXb+MTzkGeCvweuCfJO1dFl8IvJOqV9otklbYXjsZnyNiIhL0o7Hmzp0LVRMktjdJGhqDMpolwHLbPwEekLSOaqAhwDrb9wNIWl7WTdCPaSf33olgqzEoAEvLbUSWtYwczxiU6HsJ+tF4I4xBuQjYC1gAbADOq+N9MgYlpoM070TTiWFjUGw/9rOF0iXAN8vLscaaZAxK9IWc6UdjVR3NeAPDxqAMDTos3g3cVZ6vAI6RNEvSnlQ3FbyZqhvyfEl7StqW6mLvikn4CBETljP9aKwbbrgB4LXAO4aNQTlW0gKqbpwPAqcC2F4j6QqqC7SbgdNsvwggaSnVYMMZwDLbaybxo0S0LUE/GuuQQw4BuNX2wmGLVo62je1zgHNGSF851nYR00WadyIiGiRBPyKiQRL0IyIaJEE/IqJBEvQjIhokQT8iokES9CMiGiRBPyKiQRL0IyIaJEE/IqJBEvQjIhokQT8iokES9CMiGiRBPyKiQRL0IyIaJEE/IqJBEvQjIhokQT8iokES9CMiGiRBPyKiQRL0o7HWr18PsLektZLWSPowgKRdJK2SdF/5u3NJl6TPSVon6U5JBw7lJen4sv59ko6fmk8UMb5xg76kZZIel3RXS9rZkh6RdHt5HNWy7IzypbhX0hEt6YtL2jpJp9f/USImZubMmQAP294XWAScJmlf4HTgWtvzgWvLa4AjgfnlcQpwEVT/JICzgIOBg4Czhv5RREw37ZzpfxlYPEL6BbYXlMdKgPKFOQZ4a9nmC5JmSJoBXEj1pdkXOLasGzFl5s6dC/A8gO1NwN3AbsAS4LKy2mXA0eX5EuArrtwI7CRpLnAEsMr2RttPA6sY+TsTMeXGDfq2/xnY2GZ+S4Dltn9i+wFgHdWZz0HAOtv32/4psLysGzEtSJoHHADcBOxqe0NZ9Ciwa3m+G7C+ZbOHS9po6cPf4xRJqyWtfuKJJ2rd/4h2ddOmv7S0ay5r+Snb1ZcC8sWIySdpe+BrwEdsP9e6zLYB1/E+ti+2vdD2wjlz5tSRZcSEdRr0LwL2AhYAG4Dz6tqhfDFikokq4H/V9lUl7bHSbEP5+3hJfwTYo2Xb3UvaaOkR005HQd/2Y7ZftP0ScAlV8w3kSxF9pDqJ5w3A3bbPb1m0AhjqgXM88PWW9ONKL55FwLOlGega4HBJO5dfvYeXtIhpZ2YnG0ma29Lm+W5gqGfPCuBvJZ0PvJ6ql8PNVGdT8yXtSRXsjwHe182OR3TrhhtuAHgt8A5Jt5fkM4FzgSsknQw8BLy3LFsJHEV1rep54EQA2xslfQK4paz3cdvtXgeLmFTjBn1JlwNvB2ZLepiqa9rbJS2gaut8EDgVwPYaSVcAa4HNwGm2Xyz5LKU6+5kBLLO9pvZPEzEBhxxyCMCttheOsPiw4Qmlff+0kfKyvQxYVusORvTAuEHf9rEjJF86xvrnAOeMkL6S6kwpIiKmSEbkRkQ0SIJ+RESDJOhHRDRIgn5ERIMk6EdENEiCfkREgyToR0Q0SIJ+RESDJOhHRDRIgn5ERIMk6EdENEiCfkREgyToR0Q0SIJ+RESDJOhHRDRIgn5ERIMk6EdENEiCfkREgyToR0Q0SIJ+NNZJJ50EsL+ku4bSJJ0t6RFJt5fHUS3LzpC0TtK9ko5oSV9c0tZJOn1yP0XExCToR2OdcMIJAPeNsOgC2wvKYyWApH2BY4C3AouBL0iaIWkGcCFwJLAvcGxZN2JamjnVOxAxVQ499FCAzbR38rMEWG77J8ADktYBB5Vl62zfDyBpeVl3bf17HNG9nOlHbG2ppDslLZO0c0nbDVjfss7DJW209K1IOkXSakmrn3jiiV7sd8S4EvQjtnQRsBewANgAnFdXxrYvtr3Q9sI5c+bUlW3EhKR5J6KF7ceGnku6BPhmefkIsEfLqruXNMZIj5h2cqYf0ULS3JaX7waGevasAI6RNEvSnsB84GbgFmC+pD0lbUt1sXfFZO5zxETkTD8a69hjjwXYB5Ckh4GzgLdLWgAYeBA4FcD2GklXUF2g3QycZvtFqo2XAtcAM4BlttdM8keJaFuCfjTW5ZdfzvLly++0vbAl+dLR1rd9DnDOCOkrgZU92MWI2qV5JyKiQcYN+qXb2uPDRi3uImmVpPvK351LuiR9roxMvFPSgS3bHF/Wv0/S8b35OBERMZZ2zvS/TDUCsdXpwLW25wPXltdQjUqcXx6nUHV/Q9IuVO2lB1MNaDmrpf9zRERMknGDvu1/BjYOS14CXFaeXwYc3ZL+FVduBHYqvSGOAFbZ3mj7aWAVW/8jiYiIHuu0TX9X2xvK80eBXcvzjFqMiJjGur6Qa9tU3dtqkVGLERG902nQf2xoEEv5+3hJH23U4lijGSMiYpJ0GvRXAEM9cI4Hvt6SflzpxbMIeLY0A10DHC5p53IB9/CSFhERk2jcwVmSLgfeDsxuGbV4LnCFpJOBh4D3ltVXAkcB64DngRMBbG+U9AmqIesAH7c9/OJwRET02LhB3/axoyw6bIR1DZw2Sj7LgGUT2ruIiKhVRuRGRDRIgn5ERIMk6EdENEiCfkREgyToR0Q0SIJ+RESDJOhHRDRIgn5ERIMk6EdENEiCfkREgyToR2OddNJJAPtnKtBokgT9aKwTTjgB4L5hyZkKNAZagn401qGHHgqweVhypgKNgZagH7GlTAUaAy1BP2IUmQo0BlGCfsSWMhVoDLQE/YgtZSrQGGjjzpwVMaiOPfZYgH2oemRmKtBohAT9aKzLL7+c5cuX32l74bBFmQo0BlaadyIiGiRBPyKiQRL0IyIaJEE/IqJBEvQjIhokQT8iokES9CMiGiRBPyKiQRL0IyIaJEE/IqJBEvQjIhqkq6Av6UFJ35d0u6TVJW3Cc4xGRMTkqOOGa79m+8mW10NzjJ4r6fTy+o/Yco7Rg6nmGD24hvePiOiJead/a8zlD577G5O0J/XpRfPOROcYjYiISdJt0DfwbUm3SjqlpE10jtEtZB7RiIje6bZ55xDbj0j6OWCVpHtaF9q2pAnNMWr7YuBigIULF9Y2P2lERHR5pm/7kfL3ceBq4CAmPsdoRERMko6DvqRXS9ph6DnV3KB3MfE5RiMiYpJ007yzK3C1pKF8/tb2P0q6hQnMMRoREZOn46Bv+35g/xHSn2KCc4xGRMTkyIjciBFk4GEMqgT9iNH9mu0FtheW10MDD+cD15bXsOXAw1OoBh5GTEsJ+hHty8DD6HsJ+hEjy8DDGEh13HsnYhBl4GEMpAT92MJ4N5hq1Y83m2pX68BDSVsMPLS9IQMPo1+leSdimAw8jEGWM/2IrWXgYQysBP2IYTLwMAZZmnciIhokQT8iokES9CMiGiRBPyKiQRL0IyIaJEE/IqJBEvQjIhokQT8iokES9CMiGiRBPyKiQXIbhj6Vu2FGRCcS9Fu0G0gTRGNQtfMdSP3vb2neiYhokJzpR0T00Hi/nib7l1OCfo+l7b2SprOI6SFBPyJqNd3ObGNLfRn0c/YcEdGZXMiNiGiQvjzTj4gYT5qZRpYz/YiIBpn0M31Ji4HPAjOAL9o+d7L3IaJu06FeZ2BVtGNSg76kGcCFwDuBh4FbJK2wvXYy9yOiTqnX9UvTTO9M9pn+QcA62/cDSFoOLAHy5Yh+1nW9zll6TBbZnrw3k94DLLb9wfL6A8DBtpe2rHMKcEp5+Wbg3jaznw08WePuJt/e5jld8n2D7TndvFk79bqkT7Rud1s+dZRv8ujPPEat19Ou947ti4GLJ7qdpNW2F9a9P8m3v/a1l/l2a6J1u9vPUUc5JI/By2Oye+88AuzR8nr3khbRz1Kvo29MdtC/BZgvaU9J2wLHACsmeR8i6pZ6HX1jUpt3bG+WtBS4hqpr2zLba2rKfsJNQsl3SvPsx3xH1MN63e3nqKMckseA5TGpF3IjImJqZURuRESDJOhHRDTItOuyGTFRkl4JvKm8XGf7hancn4i61VnHc6Y/xST9h6neh6nWaRlIminp01S3PrgM+AqwXtKnJW1T5z5G91LXt9ROefSijvdl0Jd0RBkFOTz9PZLe2UW+MyRt3/J6kaRDy2OHTvMteb2t7N/Pldf7Sfpb4IYu8qy9HPqsDP4C2AXY0/Yv2j4Q2AvYCfhMN/s62SS9v4zkHZ7+AUnva2P7rutCXce+2+PcbVmUdQelPOqv47b77lEKa84I6bOB73WR72eAP2x5/QDwDWAV8Kku8v0L4G7gcqo+3Z8EHgU+DLxyOpVDP5UBcB+lB9qw9BnAfVNVPzv8LDcB24+Q/mrg1smoC3Uc+zqOc7dlMUjl0Ys63q9t+rNsPzE80faTkl7dRb6HAb/U8voZ2++SJOD/dpHvbwAH2H5B0s7AeuDnbT/YRZ7Qm3LopzKwyzdgWOKLkvqtL/I2tn84PNH2j9r8GV9HXajj2NdxnLstCxic8qi9jvdl8w6wo6St/mGVCrFdF/m+wvbmltd/BFWpA9uPvElbXnC58GL7aar/0A92kd+QXpRDP5XBWknHDU+U9H7gni7znmzbjRSMSjPCtm1sX0ddqOPY13Gcuy0LGJzyqL2O9+uZ/lXAJZKW2v4RQGl7+2xZ1qltJe1gexOA7W+XvF8DvLKLfN8oqXVY/p6tr23/pw7z7UU59FMZnAZcJekk4NaStpDqS/3ujvd0alwKXCnpQ7YfApA0j+o+/Ze2sX0ddaGOY1/Hce62LGBwyqP2Ot6XI3LLf/BPAh8EHgJEdcOrS4E/tf3vHeb7+8CvAx+y/a8l7Q3ARcB3bHd04UTSr4613PZ1HeZbezn0WxmUvN8BvLW8XGv72k7zmkqSPgScQXUWKWATcK7ti9rYtuu6UMexr+s4d1MWZftBK4/a6nhfBv0hkrZjy76rP64hzw8BZ1JdNAL4IROobKPkuaPt50ZZ9h+GKlQX+ddaDv1YBoNkqGfI0BnmBLftqi50e+zrPs7dlEXZfqDKoxadXP2d6gfVxZXXtbw+Dvg68Dlgl5reYwdgh5ryuq3l+bWjLZtu5dAPZTBID+Dnqfpiry6Py4BfmIq60Omxr+s4d1MWg1gedT769ULuXwE/BZB0KHAu1aCFZ+nyDnalb+5s25tsb5K0raRTJN3dTbYtz3cZY9lE9aQc+qwMBoKkJcDVwHXASeVxHVV77pI2sqilLtRw7Ls+zjWUBQxQedRuKv7TdPsA7mh5fiFwdsvr27vI9xiqSvFvVJXscKqRcFcDB3aR720jPe/2v30vyqHfymBQHsAdwLwR0ue1Hude1oU6jn0dx7nbshi08qj70a+9d2ZImumqO9VhvDzvKHTXI+lPgF+0vU7SgcD3gPfY/kYXeQL8XLkopJbnlNfdzM/ai3LotzIYFDM9Qlc+2w+qvb7pddSFOo59Hce527KAwSqPWvVr0L8cuE7Sk8CPKQMlJL2J6j9zp35qex2A7dsk3VdDsAO4hKpNcPhzgC92kW8vyqHfymBQbB7pwl7pLbJ5lG1a1VEX6jj2dRznbssCBqs8atW3vXckLQLmAt/2y/1w96Yavn1bh3k+DJzfkvT7ra9tn7/VRhPLf7btsWaw7yTPWsuhH8tgEEg6Gvg08Ods2R/7dOB021e3kUdXdaHOY9/Nca6jLEo+A1EedevLoC9p+AWRLdje2GG+Z42T78c6zPc3gS8B/w68BLzX9v/rJK9h+dZeDv1WBoNE0v7AR3m5P/Ya4Dzbd7Sxbdd1oY5jX9dx7qYsyvYDVR516teg/wBgRr76bdtvnORdGpOkO6kO9j2SDgY+bXvMQRtt5ts35dCrMmgCSf9qe8zb8E6XutDr49xOWZT1GlEenejLNn3be/Yqb0lHUo0E3LckraG6o97KLrLdbPseANs3qctbFA/pVTn0Uxk0xLhd++qqCzUc+14f57a6OTaoPCasL4N+uZI+qi7a9H8HOBX4Q6oBIVC1JZ4raXfbnfZ9b71qv9XrTtvJe1EO/VYGDTHuz/E66kJNx77Xx7mtpokGlceE9WvzzkvAXcDQhZHW//62/Y4O810LHDK8vU/Sa4Hrbb+lw3x71U5eezn0WxkMimGBYYtFwB/bHrONuo66UMexr6kdvKuyKHkMTHnUrS/P9Kmuor+HqivWcuBqj3D/7Q5opAs8tp+SOh8818MD24ty6LcyGBRj/ez/bBvb11EXuj72NR3nbssCBqs86uUpGBFW1wN4I9XNkG4CrgAWdJnfTcD+I6TvD9xc877XNhqvznLo1zLIo/u60KtjP5XHOeWx9aNfz/QBsH2/pK9T3Vv6A8DewO1dZPlRYIWkL7Fl/+Djgfd3s6N+PH8AAALpSURBVK8jqO2+GzWXQ1+WwSCSdJurOVHb1mVd6NWx7/o4d1IWMLjl0Y2+DPqS3kh1X4wlVNOPLQf+3F3eUtj29ZIOopq44ISSvBZYZPvRbvIewbe6zaAX5dBvZTDg2g4OddSFHh77Oo7zhAJlA8qjY/18IfdOqlulPsewK/rug54gkmYDT7mLAzAI5RCjk/RJ23/S5rrTti7UVNfbLouy/rQtj6nWl2f6wMd5+SB2M2/ruDr9WTksj0VUt3bdCHwC+GtgNvAKScfZ/scOs56UcqipDDYxcnc7UfWm2LGb/AfUX0pSm8GyJ3Vhose+V3V9IgG/mC7lMe3qfV+e6U8mSf9i+4Au81hNdTHpNVT38j7S9o2S9gEu7zb/XqujDGJsYwVLoJsTg273a0LHvo66Ph0D5c92YAC+C/06icpWJHU0IKsNdbS/zbT9bdv/G3jU9o0ALiP16tSjckjbe+99nuoGY5cD3wE+aPt1wKHA/+wkw5rqwkSPfdd13fYOtncc4bFDNwF/ispj2hmYoE/vroj/pbrpoF55qeX58AtJdf/Uqr0cOvhpHRPXixODOurCROv/ZNb1ieq6PAbhu9CvbfojqaM3TK/a3veX9BxVpduuPKe8fmWXuz1cV+UwnX9aD7heBMsJ1YWa6v9k1vWJmmh5DOR3YWDa9GvqIdDXbe/RvyS9CPyIEiyB54cWAa+03e6MUa15Tug7kfrfDH3ZvCNpkaT/I+kqSQdIuovqPhuPSVrcRdaT1vZeB0mbJD03wmNTyxlW9AHbM1rarWcOa8ceN+DX9J3oq/o/lnw3RtevzTuf5+Uzku8w7IwE6LQZZjq3R27F9pTfpjWmjTq+E31V/8eS78bo+rJ5R9LttheU53e75W533XSp6sVP7IjJUMd3IvW/Gfr1TL8nZyS2Z3S6bcQU6/o7kfrfDP16pp8zkogW+U5Eu/oy6EdERGf6svdORER0JkE/IqJBEvQjIhokQT8iokH+P9jLfMfg4epYAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RNj4nxkl7nEQ"
      },
      "source": [
        "In the following plots, I remove the `'O'` tag to get a higher resolution view of the distibution between entities and evaluate their differences. One of the biggest differences is that the `ORG` tags are far more prevalent in the tagset provided by the hugging face pipeline than the original. Aside from that, it seems that the `PER` labels are relatively consistent and that the difference in counts is likely due to the subtokenization procedure."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 324
        },
        "id": "fAoJzzdq4DBQ",
        "outputId": "b3bc8ae9-454c-4c00-92c6-100ce58417af"
      },
      "source": [
        "fig, axes = plt.subplots(nrows=1, ncols=2)\n",
        "\n",
        "original_s.loc[original_s != 'O'].value_counts().sort_index().plot(kind = 'bar', title = 'Original', ax=axes[0])\n",
        "predicted_s.loc[predicted_s != 'O'].value_counts().sort_index().plot(kind = 'bar', title = 'HuggingFace', ax=axes[1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7f1ffa65cb10>"
            ]
          },
          "metadata": {},
          "execution_count": 18
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEhCAYAAACEF+AUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dfbxdVX3n8c+XZ+QpQG5jStCgRCm2JdAUY3WoJaKAtaEzyGBbiAxOdAodHZxqtA/ijDrgKFRGy6tglNAqlCKUKNQxBcRhhgcDhmd5ETGUpIEEJBBA0cB3/tjrksPl3Kdz97n3nJ3v+/U6r7P32vvstc5ea/3OPmvvs49sExERzbLdVBcgIiLql+AeEdFACe4REQ2U4B4R0UAJ7hERDZTgHhHRQAnuPUbSxyV9ue51x7AtSzqwjm1FjETS05JeM9XlaLoE9y6T9F5Jd0l6VtIjks6XNG249W1/xvb7xrLt8awb0UrSGklvG5L2Xkk3djtv27vbfnCi2ynlfb58WAw+vlhHGZsgwb2LJH0YOBv4U2AvYD7wamCFpJ3arL/D5JYwou/dVD4sBh+nT3WBekWCe5dI2hP4JPAntr9t+xe21wAnALOBP5J0pqTLJf2dpKeA95a0v2vZzsmSHpL0uKS/aD3ial1X0uwytLJI0r9IekzSn7Vs53BJN0naJGm9pC+2+4CJgJcP00m6SNKnWuY/UtrRv0p6X+v6kvaV9E1JT0n6vqRPtX4jGLLuRZK+JOlqSZsl3SLptS3rvl3S/ZKelPTXkm6QNOK3VUmnSLqvbO9BSe8fsnyhpFWlfD+SdHRJ30vS0vK+1pVybz/RfTlVEty757eAXYArWhNtPw1cAxxVkhYClwPTgK+1rivpYOCvgT8EZlId/e83Sr5vAV4PLAD+UtKvlPTngf8CTAfeVJb/cQfvK7ZxJRieAbwNOBB465BVvgQ8A7wSWFQeIzmR6kBob2A18OmSz3SqvvExYF/gfqp+NZoNwO8CewKnAOdKOqxs83DgYqpv09OAI4A15XUXAVvKezoUeDvQt8OeCe7dMx14zPaWNsvWl+VQfa38R9sv2P7pkPWOB75p+0bbPwf+EhjtZkCftP1T23cAdwCHANi+zfbNtreUbxB/A/x2Z28tGuIfyze5TZI2UR1IjMUJwFdt32P7WeDMwQXlSPffAZ+w/azte4Flo2zvStu3lr7yNWBuST8WuMf2FWXZecAjQ147v/U9SJpv+2rbP3LlBuA7wL8p658KfMX2itLn1tn+oaQZJb8P2X7G9gbgXKoPnr6UMd7ueQyYLmmHNgF+ZlkO8PAI2/jl1uW2n5X0+Cj5tjb+Z4HdASS9DjgHmAe8gqrubxvtTUSjHWf7nwdnJL2XsR2p/jKwsmW+tQ0PULWth4dZ3k7bNsvL278lrR3y2pttv6U1QdIxwCeA11EdwL4CuKss3p/qm/NQrwZ2BNZLGkzbbgxl71k5cu+em4DngH/bmihpd+AY4NqSNNKR+HpgVstrd6X6etqJ84EfAnNs7wl8HNDIL4lt2LNUQXHQK1umX9IuqQLmoI1UQxvDLR+Poe1fQ7b7MpJ2Br4BfA6YYXsaVTAfbOsPA69t89KHqfrrdNvTymNP22/osOxTLsG9S2w/STWO+L8kHS1pR0mzgcuAtcDfjmEzlwPvkvRb5eTnmXQekPcAngKelnQQ8J863E5sG1YBfyBp+zLG3jqEdxlwiqRfkfQK4C8GF9h+nuo805mSXlHa2skdluFq4NckHVeuJDuNl37ItLMTsDPlQ6Ycxb+9ZfnSUvYFkraTtJ+kg2yvpxq++bykPcuy10rq26HLBPcusv1ZqiPkz1EF1luojhAW2H5uDK+/B/gT4FKqo5inqU4WjfraNv4r8AfAZuBC4O872EZsOz4IvAvYRHVC/x8HF9j+J6rx7+upToDeXBYNtsvTqU7+P0J1EHMJHbRZ248B7wY+CzwOHEw1HDTstmxvBv4z1QfQE1RtfnnL8lspJ1mBJ4EbqIZkoPoQ2gm4t7z2cqoh1L6k/FlH/yhDOpuohlZ+PNXliQAoV2TdDezc7gICSWcDr7Q92lUzo+WzHdW33j+0ff1EtrUtyJF7j5P0rvL1djeqbwB3sfXSrYgpIen3Je0saW+qH+p9czCwSzpI0q+rcjjVFSpXdpjPOyRNK2Ppg+eJbh7lZUGCez9YCPxrecwBTnS+bsXUez/VEOGPqH5D0XoOZw+qcfdnqIb/Pg9c1WE+byp5PEY1THRcm0uGo40My0RENFCO3CMiGqgnfsQ0ffp0z549e6qLEQ112223PWZ7YCryTtuObhqpbfdEcJ89ezYrV64cfcWIDkh6aKryTtuObhqpbWdYJiKigRLcIyIaKME9tlmSXl/u6z34eErShyTtI2mFpAfK895lfUk6T9JqSXcO3kY2oheNGtwl7SLpVkl3SLpH0idL+kWSftzSMeaW9HSA6Au277c91/Zc4DeobpZ1JbAEuNb2HKobvC0pLzmG6rcGc4DFVDdji+hJYzmh+hxwpO2nJe0I3Cjpn8qyP7V9+ZD1WzvAG6k6wBvrKnBElywAfmT7IUkL2foHFMuA7wIfpfpB2cXlR2Q3l19Oziw3nYroKaMeuZcb3j9dZncsj5F++fRiB7B9MzBNUt/efCe2GSdS3eAKqlvFDgbsR4AZZXo/Xnp/77W0+WcsSYslrZS0cuPGjd0qb8SIxjTmXm77uYrq58YrbN9SFn26DL2cW+79AOkA0WfK7ZR/D/iHocvKUfq4fsZt+wLb82zPGxiYksvrI8YW3G0/X8YlZwGHS/pVqv81PAj4TWAfqq+tY5YOED3kGOB224+W+UcHv22W5w0lfR0v/eOJWSUtoueM62oZ25uo7uF8tO31ZejlOeCrwOFltXSA6DfvYeuQDFT3/x68Pe0itt70ajlwcrloYD7wZMbbo1eNekJV0gDwC9ubyt+8HQWcPXgiqfz11XFU93OGqgOcLulSqhOp6QANM3vJ1eN+zZqz3tmFkkxcuZXyUVR3ORx0FnCZpFOBh6j+EBqqv2s7luoPKp6l+tOH2EaNpR9MZbsfy9UyM4Flqv7VfDvgMtvfknRdCfyi+kuuD5T10wGib9h+hiH/S2v7caqrZ4aua6q/eovoeaMGd9t3Aoe2ST9ymPXTASIiplh+oRoR0UAJ7hERDZTgHhHRQAnuERENlOAeEdFACe4REQ2U4B4R0UAJ7hERDZTgHhHRQAnuERENlOAeEdFACe4REQ2U4B4R0UAJ7hERDZTgHhHRQAnuERENlOAeEdFACe4REQ2U4B4R0UAJ7hERDZTgHts0SdMkXS7ph5Luk/QmSftIWiHpgfK8d1lXks6TtFrSnZIOm+ryRwxn1OAuaRdJt0q6Q9I9kj5Z0g+QdEtp6H8vaaeSvnOZX12Wz+7uW4iYkC8A37Z9EHAIcB+wBLjW9hzg2jIPcAwwpzwWA+dPfnEjxmYsR+7PAUfaPgSYCxwtaT5wNnCu7QOBJ4BTy/qnAk+U9HPLehE9R9JewBHAUgDbP7e9CVgILCurLQOOK9MLgYtduRmYJmnmJBc7YkxGDe6lIT9dZncsDwNHApeX9KEdYLBjXA4skKTaShxRnwOAjcBXJf1A0pcl7QbMsL2+rPMIMKNM7wc83PL6tSXtJSQtlrRS0sqNGzd2sfgRwxvTmLuk7SWtAjYAK4AfAZtsbymrtDbyFztAWf4ksG+bbaYDxFTbATgMON/2ocAzbB2CAaqDG6qDmTGzfYHtebbnDQwM1FbYiPEYU3C3/bztucAs4HDgoIlmnA4QPWAtsNb2LWX+cqpg/+jgcEt53lCWrwP2b3n9rJIW0XPGdbVMGY+8HngT1XjjDmVRayN/sQOU5XsBj9dS2oga2X4EeFjS60vSAuBeYDmwqKQtAq4q08uBk8tVM/OBJ1uGbyJ6yg6jrSBpAPiF7U2SdgWOojpJej1wPHApL+8Ai4CbyvLrylfbiF70J8DXytVeDwKnUB30XCbpVOAh4ISy7jXAscBq4NmybkRPGjW4AzOBZZK2pzR629+SdC9wqaRPAT+gXHFQnv9W0mrgJ8CJXSh3RC1srwLmtVm0oM26Bk7reqEiajBqcLd9J3Bom/QHqcbfh6b/DHh3LaWLiIiO5BeqERENlOAeEdFACe4REQ2U4B4R0UAJ7hERDZTgHhHRQAnuERENlOAeEdFACe4REQ2U4B4R0UAJ7hERDZTgHhHRQAnuERENlOAeEdFACe4REQ2U4B4R0UAJ7hERDZTgHhHRQAnuERENlOAeEdFACe6xTZO0RtJdklZJWlnS9pG0QtID5Xnvki5J50laLelOSYdNbekjhjdqcJe0v6TrJd0r6R5JHyzpZ0paVzrFKknHtrzmY6UD3C/pHd18AxE1+B3bc23PK/NLgGttzwGuLfMAxwBzymMxcP6klzRijHYYwzpbgA/bvl3SHsBtklaUZefa/lzrypIOBk4E3gD8MvDPkl5n+/k6Cx7RRQuBt5bpZcB3gY+W9IttG7hZ0jRJM22vn5JSRoxg1CN32+tt316mNwP3AfuN8JKFwKW2n7P9Y2A1cHgdhY3oAgPfkXSbpMUlbUZLwH4EmFGm9wMebnntWtr0BUmLJa2UtHLjxo3dKnfEiMY15i5pNnAocEtJOr2MPX5lcFySdIDoL2+xfRjVkMtpko5oXViO0j2eDdq+wPY82/MGBgZqLGrE2I05uEvaHfgG8CHbT1GNN74WmAusBz4/nozTAaIX2F5XnjcAV1J9y3xU0kyA8ryhrL4O2L/l5bNKWkTPGVNwl7QjVWD/mu0rAGw/avt52y8AF7J16CUdIPqCpN3KeSQk7Qa8HbgbWA4sKqstAq4q08uBk8tVM/OBJzPeHr1q1BOqkgQsBe6zfU5LeuuJpN+n6hRQdYCvSzqH6oTqHODWWksdUY8ZwJVVE2cH4Ou2vy3p+8Blkk4FHgJOKOtfAxxLdR7pWeCUyS9yxNiM5WqZNwMnAXdJWlXSPg68R9JcqvHINcD7AWzfI+ky4F6qK21Oy5Uy0YtsPwgc0ib9cWBBm3QDp01C0SImbNTgbvtGQG0WXTPCaz4NfHoC5YqIiAnIL1QjIhoowT0iooES3CMiGijBPSKigRLcIyIaKME9IqKBEtwjIhoowT0iooES3CMiGijBPSKigRLcIyIaKME9IqKBEtwjIhoowT0iooES3CMiGijBPSKigRLcIyIaKME9IqKBEtwjIhoowT0iooES3GObJ2l7ST+Q9K0yf4CkWyStlvT3knYq6TuX+dVl+eypLHfESEYN7pL2l3S9pHsl3SPpgyV9H0krJD1Qnvcu6ZJ0XukAd0o6rNtvImKCPgjc1zJ/NnCu7QOBJ4BTS/qpwBMl/dyyXkRPGsuR+xbgw7YPBuYDp0k6GFgCXGt7DnBtmQc4BphTHouB82svdURNJM0C3gl8ucwLOBK4vKyyDDiuTC8s85TlC8r6ET1n1OBue73t28v0ZqojnP14aUMf2gEuduVmYJqkmbWXPKIefwV8BHihzO8LbLK9pcyvpWrvlOeHAcryJ8v6LyFpsaSVklZu3Lixm2WPGNa4xtzLGOOhwC3ADNvry6JHgBll+sUOULR2jtZtpQPElJL0u8AG27fVuV3bF9ieZ3vewMBAnZuOGLMxB3dJuwPfAD5k+6nWZbYNeDwZpwNED3gz8HuS1gCXUg3HfIHq2+YOZZ1ZwLoyvQ7YH6As3wt4fDILHDFWYwruknakCuxfs31FSX50cLilPG8o6S92gKK1c0T0DNsfsz3L9mzgROA6238IXA8cX1ZbBFxVppeXecry68qBTUTPGcvVMgKWAvfZPqdlUWtDH9oBTi5XzcwHnmwZvonoBx8FzpC0mmpMfWlJXwrsW9LPYOtFBBE9Z4fRV+HNwEnAXZJWlbSPA2cBl0k6FXgIOKEsuwY4FlgNPAucUmuJI7rA9neB75bpB4HD26zzM+Ddk1qwiA6NGtxt3wgMd7nXgjbrGzhtguWKiIgJyC9UIyIaKME9IqKBEtwjIhoowT0iooES3CMiGijBPSKigRLcIyIaKME9IqKBEtwjIhoowT0iooES3CMiGijBPSKigRLcIyIaKME9IqKBEtwjIhoowT0iooES3CMiGmgsf7MXfWT2kqvHtf6as97ZpZJExFTKkXtERAMluEdENFCGZRj/UAZkOKMJJO0CfA/YmaovXG77E5IOAC4F9gVuA06y/XNJOwMXA78BPA78e9trpqTwEaMY9chd0lckbZB0d0vamZLWSVpVHse2LPuYpNWS7pf0jm4VPKIGzwFH2j4EmAscLWk+cDZwru0DgSeAU8v6pwJPlPRzy3oRPWkswzIXAUe3ST/X9tzyuAZA0sHAicAbymv+WtL2dRU2ok6uPF1mdywPA0cCl5f0ZcBxZXphmacsXyBJk1TciHEZNbjb/h7wkzFubyFwqe3nbP8YWA0cPoHyRXSVpO0lrQI2ACuAHwGbbG8pq6wF9ivT+wEPA5TlT1IN3Qzd5mJJKyWt3LhxY7ffQkRbEzmherqkO8uwzd4l7cXGX7R2jJdIB4heYPt523OBWVQHIgfVsM0LbM+zPW9gYGDCZYzoRKfB/XzgtVTjlOuBz493A+kA0UtsbwKuB94ETJM0eLHBLGBdmV4H7A9Qlu9FdWI1oud0FNxtP1qOeF4ALmTr0MuLjb9o7RgRPUXSgKRpZXpX4CjgPqogf3xZbRFwVZleXuYpy6+z7ckrccTYdRTcJc1smf19YPBKmuXAiZJ2LpeTzQFunVgRI7pmJnC9pDuB7wMrbH8L+ChwhqTVVGPqS8v6S4F9S/oZwJIpKHPEmIx6nbukS4C3AtMlrQU+AbxV0lyqKwvWAO8HsH2PpMuAe4EtwGm2n+9O0SMmxvadwKFt0h+kzYUAtn8GvHsSihYxYaMGd9vvaZO8tE3a4PqfBj49kUJFRMTE5PYDERENlOAeEdFACe4REQ2U4B4R0UAJ7hERDZTgHhHRQAnuERENlOAeEdFACe4REQ2U4B4R0UAJ7hERDZTgHhHRQKPeOCxiKsxecvW4X7PmrHd2oSQR/SlH7hERDZTgHhHRQAnuERENlOAeEdFACe4REQ2Uq2UmSa7+iIjJlCP3iIgGSnCPbZak/SVdL+leSfdI+mBJ30fSCkkPlOe9S7oknSdptaQ7JR02te8gYnijBndJX5G0QdLdLWlp/NEEW4AP2z4YmA+cJulgYAlwre05wLVlHuAYYE55LAbOn/wiR4zNWI7cLwKOHpKWxh99z/Z627eX6c3AfcB+wEJgWVltGXBcmV4IXOzKzcA0STMnudgRYzJqcLf9PeAnQ5LT+KNRJM0GDgVuAWbYXl8WPQLMKNP7AQ+3vGxtSRu6rcWSVkpauXHjxq6VOWIknV4tM97Gv54hJC2mOrrnVa96VYfFiJg4SbsD3wA+ZPspSS8us21JHs/2bF8AXAAwb968cb12Ko3liq5cwdU/JnxC1baBcTdg2xfYnmd73sDAwESLEdERSTtSBfav2b6iJD86+I2zPG8o6euA/VtePqukRfScToN7Gn/0PVWH6EuB+2yf07JoObCoTC8CrmpJP7lcODAfeLLlG2xET+k0uKfxRxO8GTgJOFLSqvI4FjgLOErSA8DbyjzANcCDwGrgQuCPp6DMEWMy6pi7pEuAtwLTJa0FPkHV2C+TdCrwEHBCWf0a4Fiqxv8scEoXyhxRC9s3Ahpm8YI26xs4rauFiqjJqMHd9nuGWZTGHxHRo3r+3jLjvSdLzuZHROT2AxERjZTgHhHRQAnuEREN1PNj7hERTdeNXwfnyD0iooES3CMiGijBPSKigRLcIyIaKME9IqKBEtwjIhoowT0iooES3CMiGijBPSKigRLcIyIaKME9IqKBcm+ZiKhdN+6VEuOTI/eIiAZKcI+IaKAE94iIBkpwj22WpK9I2iDp7pa0fSStkPRAed67pEvSeZJWS7pT0mFTV/KI0U0ouEtaI+kuSaskrSxpbTtHRA+6CDh6SNoS4Frbc4BryzzAMcCc8lgMnD9JZYzoSB1H7r9je67teWV+uM4R0VNsfw/4yZDkhcCyMr0MOK4l/WJXbgamSZo5OSWNGL9uDMsM1zki+sEM2+vL9CPAjDK9H/Bwy3prS1pET5pocDfwHUm3SVpc0obrHC8habGklZJWbty4cYLFiKifbVO18XFJ245eMNHg/hbbh1GNR54m6YjWhSN1DtsX2J5ne97AwMAEixFRm0cHh1vK84aSvg7Yv2W9WSXtZdK2oxdMKLjbXleeNwBXAoczfOeI6AfLgUVlehFwVUv6yeWqmfnAky3fUCN6TsfBXdJukvYYnAbeDtzN8J0joqdIugS4CXi9pLWSTgXOAo6S9ADwtjIPcA3wILAauBD44ykocsSYTeTeMjOAKyUNbufrtr8t6fvAZaWjPAScMPFiRtTP9nuGWbSgzboGTutuiSLq03Fwt/0gcEib9Mdp0zkiImLy5BeqERENlOAeEdFAuZ97RGwztqX7zOfIPSKigRLcIyIaKME9IqKBEtwjIhoowT0iooES3CMiGijBPSKigRLcIyIaKME9IqKB8gvViD62Lf3iMsYnwT0ielo+wDqTYZmIiAZKcI+IaKAE94iIBkpwj4hooJxQjZhkOUEYkyFH7hERDZTgHhHRQF0L7pKOlnS/pNWSlnQrn4jJlHYd/aIrwV3S9sCXgGOAg4H3SDq4G3lFTJa06+gn3TpyPxxYbftB2z8HLgUWdimviMmSdh19Q7br36h0PHC07feV+ZOAN9o+vWWdxcDiMvt64P5xZjMdeKyG4k51HpOVT1Py6CSfV9semGimY2nXJX28bbvO/ZZtbVvbGrZtT9mlkLYvAC7o9PWSVtqeV2ORpiSPycqnKXlMZj6dGm/brvP9ZFvZ1qBuDcusA/ZvmZ9V0iL6Wdp19I1uBffvA3MkHSBpJ+BEYHmX8oqYLGnX0Te6Mixje4uk04H/DWwPfMX2PTVn0/GQTo/lMVn5NCWPycznJbrYrut8P9lWtgV06YRqRERMrfxCNSKigRLcIyIaKME9IqKBEtx7iKRXTXUZ+kn2V/9JnY3PRPZXXwR3Se8ovw4cmn68pKNqymN7Sbu3zM+XdER57FFHHi3bflMp+y+V+V+X9HXg/9a0/eyvHiXpj8ovW4emnyTpD8a5rdrque76rKvOsr8m0MZt9/yjvMGBNunTgZtqyuNzwEda5n8MfBNYAZxd43v5n8B9wCVU101/CngE+CCwS/bX5O+vyXwAtwC7t0nfDbhtquq5zvqss86yvzpv4/3yT0w72944NNH2Y5J2qymPBcBvtsxvsv0uSQL+T015ALwTONT2zyTtDTwM/KrtNTXmkf3Vu3a0/fTQRNvPSNpxnNuqs57rrM866yz7q0N9MSwD7CnpZR9EpXJ3rSmP7WxvaZn/KICrj9bd27+kIz+z/bOy7SeAB7oQqLK/eteu7QJJ+Sq/0zi3VWc911mfddZZ9leH+uXI/QrgQkmn234GoIx3faEsq8NOkvawvRnA9ndKPnsBu9SUB8BrJLX+ZP2A1nnbv1dDHtlfvWspcLmkD9h+CEDSbKr7xC8d57bqrOc667POOsv+6rCN98UvVMun7aeA9wEPAaK6gdNS4C9s/6KGPM4A3gZ8wPa/lLRXA+cD19n+3ETzKNv87ZGW276hhjyyv3qYpA8AH6M6whOwGTjL9vnj3E5t9VxnfdZdZ9lfnbXxvgjugyTtChxYZlfb/mnN2/8A8HGqkzUAT9NBIxoljz1tPzXMslcNNpSa8sr+6mGDV1UMHv1NYDu11HNd9dmtOsv+GqdOz8RO5oPqxMUrW+ZPBq4CzgP26UJ+ewB7dOm93N4yfe1wy7K/Jm9/TfYD+FVgGbCyPJYBv9Yr9TzR+qy7zrK/Ottuv5xQ/Rvg5wCSjgDOAi4GnqTGO7GVa1en295se7OknSQtlnRfXXlQfRUctM8IyyYi+6tHSVoIXAncAPyH8rgBuKIsG49a67nG+qytzrK/JtDGO/1UmMwHcEfL9JeAM1vmV9WUx4lUlfyvVI3n7cBaqoZ1WI3v5fZ20xP9lM7+6o8HcAcwu0367NZ6m+x6rrM+66yz7K/O23i/XC2zvaQdXF16tICt/08J9V3x8+fAb9heLekw4CbgeNvfrGn7g36pnIxRyzRlfsL/81lkf/WuHdzmMjfbazq4brvOeq6zPuuss+yvDvVLcL8EuEHSY8BPKT8SkHQg1adnHX5uezWA7dslPdCFQAVwIdUY3dBpgC/XlEf2V+/a0u4kWbnSYsswrxlOnfVcZ33WWWfZXx3qm6tlJM0HZgLf8dZrVF9H9dPk22vY/lrgnJakM1rnbZ/zshdNLL/ptuv6p/R228/+6kGSjgM+C3wGuK0kzwOWAEtsXznO7dVSz92ozzrqLPurc30R3CUNPcnwErZ/UkMenxglj09ONI+Sz+8CXwV+AbwAnGD7/9Wx7ZY8sr96mKRDgA8DbyhJ9wCft33HOLdTWz3XWZ9111n2V2f6Jbj/GDDtzxzb9msmuUgdk3QnVeX9UNIbgc/aHvFHDB3kkf3VhyT9i+0x3+K1V+t5suos+2tkfTHmbvuAychH0jFUv4Q7uCTdQ3WHt2tqzGaL7R8C2L5FNd8et2w3+6s/jeuyt7rrucb6nKw6y/4aQV8E93I2elg1jSH/R+D9wEeofigB1djeWZJm2a7r+vDWs+Evm69jrDr7q2+N62t0nfVcc31OVp1lf41Uxj4ZlnkBuBsYPNnQ+olt20fWkMe9wFuGjrtJ2he40favTDSPsr2uj1Vnf/WuIZ34JYuAP7M94rjwkG3VVs911mfN49HZXx228b44cqc6E3081eVLlwJXus09nidI7U6o2H5cqu+HkJMUjLK/etdIX7m/MM5t1VnPtdVnzXWW/dUpd/jrp6l4AK+hulHPLcBlwNwat30LcEib9EOAW7v8vrryS8vsr23jUUc9d7s+e6nOtpX91S9H7gDYflDSVVQ31j8JeB2wqqbNfxhYLumrvPR62kXAH9WUx3C6co+U7K/+IOl22yOOB4+kpnrudn3WVmfZX2PTF8Fd0muo7lzjj6MAAAKmSURBVOWwkOovqC4FPuMab2Fr+0ZJhwOnAe8tyfcC820/Ulc+w7i6zo1lf/WdjjpynfU8CfVZZ51lf41BP51QvZPq9pxPMeQsufv8iglJ04HHXVNlNH1/NY2kT9n+8w5e1zf1XGcb3xb2Vx364sgd+G9srYg6/59zRBP9+jfMNudT3Wr0J8B/B/6W6t/Xt5N0su1v15BNk/bXZtpf8iaqKxz2rDO/KfJXktRB4OtqPXdan91u450E9qJX91dX2nhfHLlPFUk/sH1ozdtcSXUyZy+qe0gfY/tmSQcBl9Sd32Tqxv5qmpECH1DXh3stOq3POtt4P32491r775c/63gZSRP+Ic4YdGNsdwfb37H9D8Ajtm8GcPmFWrf08f5qmi9S3QTrEuA64H22XwkcAfyPiW685nrutD5ra+O297C9Z5vHHnUE9h7ZX13Rt8Gdybli4q9U50XblRdapoeeyOnm16iu768JfF3elnT7w73Oeu60/U9VG+9Ebfur19p/v4y5t1P3FSaTMRYOcIikp6ga1a5lmjK/S015tFP3/uqbr8s9ptuBr6N6rrn9T1Ub70Sn+6vn23/fjrl34QqTxo6FR++Q9DzwDCXwAc8OLgJ2sT3efxcauv2O+kXaf/P0xbCMpPmSvivpCkmHSrqb6h4Rj0o6uqZspmQsvBskbZb0VJvH5pajqJgCtrdvGTPeYcgY8rgCe839ojHtfzjbWr/ol2GZL7L1qOI6hhxVAHUMmfTTOOGIbDf5trixVZ39ojHtfzjbWr/oi2EZSatszy3T97nljmt1XX7U7a/LEXWrs1+k/TdPvxy5d/2owvb2dWwnYhLV1i/S/punX47cc1QRMUT6RYykL4J7RESMT19cLRMREeOT4B4R0UAJ7hERDZTgHhHRQP8fHFaQ3IYcyy8AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q3N0ysPX28Rh"
      },
      "source": [
        "stripped_original = original_s.loc[original_s != 'O'].apply(lambda x: x.split('-')[-1])\n",
        "stripped_predicted = predicted_s.loc[predicted_s != 'O'].apply(lambda x: x.split('-')[-1])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 314
        },
        "id": "6GL7vDbt-IrU",
        "outputId": "1572f86d-55ce-4188-e6c0-3ac73ed4e960"
      },
      "source": [
        "original_s = pd.Series(flat_labels)\n",
        "predicted_s = pd.Series(pred_labels)\n",
        "\n",
        "stripped_original = original_s.loc[original_s != 'O'].apply(lambda x: x.split('-')[-1])\n",
        "stripped_predicted = predicted_s.loc[predicted_s != 'O'].apply(lambda x: x.split('-')[-1])\n",
        "\n",
        "fig, axes = plt.subplots(nrows=1, ncols=2)\n",
        "stripped_original.value_counts().sort_index().plot(kind = 'bar', title = 'Original Labels', ax=axes[0])\n",
        "stripped_predicted.value_counts().sort_index().plot(kind = 'bar', title = 'HuggingFace Labels', ax=axes[1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7f1ffa4006d0>"
            ]
          },
          "metadata": {},
          "execution_count": 21
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEXCAYAAABWNASkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAb1UlEQVR4nO3dfZRdVZnn8e+vE0h4TSBExCRQjAk6gVHAiDCgbSc4DQGFUaChaYl0eqI9dAsTpyGypqVtsVdYOh2hbdEgSOgFCAJKFmC3NC+Ko8QOL81LECnTQBITEiAJbyJGnvnj7Aq3LvVyb9W995za9fusdVeds8/bc6v2furcfe45WxGBmZnl5ffKDsDMzFrPyd3MLENO7mZmGXJyNzPLkJO7mVmGnNzNzDLk5J4xSR+UtLbT21p+JF0p6cI0/X5Jj3fouCFpeov3uf29dHLbTnNyb4Kkl2per0v6dc386W087ick/bhd+7c8SHqypk4+kxLRrq0+TkTcExHvaCCettZbSXdL+rN27X+kc3JvQkTs2vMCngY+XFN2dc96ksaWF6WNch9O9fNQYBbwf+pXcP0cHZzcW6CnC0PSeZI2AN/q66yl9iOmpHGSvizp6XSW9XVJOw3h2GdKekzSi5JWS/pkH+ucL+nZdGZ3ek15wzGk97YuHedxSXOajdU6JyLWAd8HDoLtde8sSU8AT6Sy4yU9KGmLpJ9IelfP9pIOkXR/+ntfB4yvWdary07SNEk3Sdok6TlJX5X0n4GvA0ekTxJb0roD1jlJfyVpvaRfSfrTob5/Sd+RtEHSVkk/knRg3Sp7Sbo9vb8fStqvZtt3pmXPp7p+Sj/H2EvSLen397ykeyRVJqdWJpAMvBXYE9gPWNDA+ouBA4CDgenAFOBzQzjuRuB4YHfgTGCJpEPr4tor7X8esFRSz0fqhmJI6/8F8N6I2A34Q+DJIcRqHSJpGjAXeKCm+ETgfcBMSYcAVwCfBCYB3wCWp+S7I/A94J8o6vR3gI/1c5wxwC3AU0AXRR36dkQ8BnwK+Gn6ZDsxbdJvnZN0DPC/gQ8BM4Cjh/Er+H7ax1uA+4Gr65afDnyBom082LNc0i7A7cA1adtTga9JmtnHMT4DrAUmA3sD5wPVeZ5LRPg1hBdFcjs6TX8QeA0YX7P8E8CP67YJigot4GXg7TXLjgD+o59jvWlfA8T1PeDsmri2AbvULL8e+OvBYkjbrk3T0yn+iRwN7FD2796vAevkS8AWimT7NWCnmro3u2bdS4Ev1G3/OPD7wAeAXwGqWfYT4MI+6sYRwCZgbB/x9Kq3DdS5K4DFNcsO6Gkz/bzfu4E/a+D3MjHtZ0Kav5LiH1DP8l2B3wHTgD8C7qnb/hvABTXb9vwe/ha4ub/4yn657611NkXEqw2uOxnYGbhPUk+ZgDHNHlTSscAFFA3h99J+H65ZZXNEvFwz/xTwtmZiiIhuSecAfwMcKOlfgIUR8atm47W2OzEi/rWfZWtqpvcD5kn6y5qyHSnqRgDrImWw5Kl+9jkNeCoitjUQ22B17m3AfQ0cc0Dp08QXgZPTMV9Pi/YCtqbp7b+LiHhJ0vPp+PsB7+vpRkrGUnyKqfclijbxg/R+lkbE4qHE3A7ulmmd+o9jL1NUZAAkvbVm2bPAr4EDI2Jiek2I4kJYwySNA24EvgzsHcVH39soGkyPPdJHzR77UpyVNRVDRFwTEUdRVP4ALmomVquE2jq6Bvhizd9+YkTsHBHXAuuBKarJwBT1pi9rgH37uUhb3yYGq3PrKf5ZDHbMwfwxcALFJ80JFN1F0LtdbD9O+kbRnhTtYg3ww7rfy64R8edvenMRL0bEZyLiPwEfARZW6VqUk3v7/DvFWe7BksZT/IcHICJeBy6j6B9/C4CkKZL+cID9SdL42hfFmdY4io/F29JZ/H/rY9vPS9pR0vsp+ue/00wMkt4haXb6Z/IqRQN9vX49G1EuAz4l6X0q7CLpOEm7AT+l6M77tKQdJH0UOKyf/fyMIikvTvsYL+nItOwZYGrqw2+k3l8PfELSTEk7U3wiHczYunaxA7Ab8BvgOYoTrL/rY7u5ko5KsX0BuDci1lBcPzhA0sfTe99B0nvTBeJeVFyQnp7+CW6l6NqpTLtwcm+TiPgFRZ/cv1J8O6H++77nAd3AvZJeSOsN9N3h/0qRVOtfn6ZoFJspzliW1223IS37FcVFo09FxM+bjGEcxYWwZ9P+3gJ8doBYreIiYiXwP4CvUtSPboo+ciLiNeCjaf55in7om/rZz++AD1Ncl3ma4gLjH6XFdwKPAhskPZvK+q1zEfF94Ctpu+70czCX0rs9fAu4iqJLZx2wCri3j+2uofjn8TzwHuBPUgwvUpwgnUrRZjZQfEod18c+ZqT4X6L4h/i1iLirgZg7Qr271czMLAc+czczy5CTu5lZhpzczcwy5ORuZpYhJ3czswxV4g7VvfbaK7q6usoOwzJ13333PRsRk8s4tuu2tdNAdbsSyb2rq4uVK1eWHYZlStKQbmNvBddta6eB6ra7ZczMMuTkbmaWISd3M7MMObmbmWXIyd3MLEMNJXcVY28+rGK8xZWpbM80zuAT6eceqVySLpHULemhuiHfzMysA5o5c/+DiDg4Imal+UXAHRExA7gjzQMcS/EozBkUY4le2qpgzcysMcPpljkBWJaml1EMvttTflUU7gUmStpnGMcxM7MmNXoTU1CMExjANyJiKcWwbuvT8g0Uo39DMZp57ViNa1PZeuxNuhbd2pHjPLn4uI4cx6zdWtlmcm4XjSb3oyJiXRoa63ZJP69dGBGREn/DJC2g6LZh332HOlSimZn1paFumYhYl35uBL5LMZ7iMz3dLennxrT6OnoPcjs1ldXvc2lEzIqIWZMnl/LYDzOzbA2a3NOgt7v1TFOML/gIxVid89Jq84Cb0/Ry4Iz0rZnDga013TdmZtYBjXTL7A18txjgm7HANRHxz5L+Dbhe0nyKwWhPSevfBsylGOD2FeDMlkdtZmYDGjS5R8Rq4N19lD8HzOmjPICzWhKdmZkNie9QNTPLkJO7mVmGnNzNzDLk5G5mliEndzOzDDm5m5llyMndzCxDTu5mZhlycjczy5CTu5lZhpzcbVST9L8kPSrpEUnXShovaX9JK9JQkddJ2jGtOy7Nd6flXeVGb9Y/J3cbtSRNAT4NzIqIg4AxwKnARcCSiJgObAbmp03mA5tT+ZK0nlklObnbaDcW2EnSWGBnihHDZgM3pOX1Q0j2DC15AzBH6XGpZlXj5G6jVhqE5svA0xRJfStwH7AlIral1XqGiYSaISTT8q3ApE7GbNYoJ3cbtSTtQXE2vj/wNmAX4JgW7HeBpJWSVm7atGm4uzMbEid3G82OBv4jIjZFxG+Bm4AjgYmpmwZ6DxO5fQjJtHwC8Fz9Tj2EpFWBk7uNZk8Dh0vaOfWdzwFWAXcBJ6V16oeQ7Bla8iTgzjQ4jVnlOLnbqBURKygujN4PPEzRHpYC5wELJXVT9Klfnja5HJiUyhcCizoetFmDGhlD1SxbEXEBcEFd8WrgsD7WfRU4uRNxmQ2Xz9zNzDLk5G5mliEndzOzDDm5m5llyMndzCxDTu5mZhlycjczy5CTu5lZhpzczcwy5ORuZpYhJ3czsww5uZuZZcjJ3cwsQ07uZmYZaji5Sxoj6QFJt6T5/SWtkNQt6TpJO6bycWm+Oy3vak/oZmbWn2bO3M8GHquZvwhYEhHTgc3A/FQ+H9icypek9czMrIMaSu6SpgLHAd9M8wJmU4xiA7AMODFNn5DmScvnpPXNzKxDGj1z/wpwLvB6mp8EbImIbWl+LTAlTU8B1gCk5VvT+r14hHgzs/YZNLlLOh7YGBH3tfLAHiHezKx9GhlD9UjgI5LmAuOB3YGLgYmSxqaz86nAurT+OmAasFbSWGAC8FzLIzczs34NeuYeEZ+NiKkR0QWcCtwZEacDdwEnpdXmATen6eVpnrT8zoiIlkZtZmYDGs733M8DFkrqpuhTvzyVXw5MSuULgUXDC9HMzJrVSLfMdhFxN3B3ml4NHNbHOq8CJ7cgNjMzGyLfoWpmliEndzOzDDm5m5llyMndzCxDTu5mZhlycjczy5CTu5lZhpzczcwy5ORuZpYhJ3czsww5uZuZZcjJ3cwsQ07uZmYZcnI3M8uQk7uZWYac3G1UkzRR0g2Sfi7pMUlHSNpT0u2Snkg/90jrStIlkrolPSTp0LLjN+uPk7uNdhcD/xwR7wTeDTxGMXrYHRExA7iDN0YTOxaYkV4LgEs7H65ZY5zcbdSSNAH4AGmIyIh4LSK2ACcAy9Jqy4AT0/QJwFVRuJdikPh9Ohy2WUOc3G002x/YBHxL0gOSvilpF2DviFif1tkA7J2mpwBrarZfm8rMKsfJ3UazscChwKURcQjwMnUDukdEANHMTiUtkLRS0spNmza1LFizZji522i2FlgbESvS/A0Uyf6Znu6W9HNjWr4OmFaz/dRU1ktELI2IWRExa/LkyW0L3mwgTu42akXEBmCNpHekojnAKmA5MC+VzQNuTtPLgTPSt2YOB7bWdN+YVcrYsgMwK9lfAldL2hFYDZxJcdJzvaT5wFPAKWnd24C5QDfwSlrXrJKc3G1Ui4gHgVl9LJrTx7oBnNX2oMxawN0yZmYZcnI3M8uQk7uZWYac3M3MMuTkbmaWISd3M7MMObmbmWXIyd3MLEODJndJ4yX9TNK/S3pU0udT+f6SVqSBC65Ld/ghaVya707Lu9r7FszMrF4jZ+6/AWZHxLuBg4Fj0nM1LgKWRMR0YDMwP60/H9icypek9czMrIMGTe5pYIKX0uwO6RXAbIqn6MGbBzToGejgBmCOJLUsYjMzG1RDfe6Sxkh6kOLRp7cDvwS2RMS2tErtoAXbBzRIy7cCk/rYp595bWbWJg0l94j4XUQcTPH86sOAdw73wH7mtZlZ+zT1bZk0vuRdwBEU40f2PFWydtCC7QMapOUTgOdaEq2ZmTWkkW/LTJY0MU3vBHyIYoT4u4CT0mr1Axr0DHRwEnBnelSqmZl1SCPPc98HWCZpDGkQg4i4RdIq4NuSLgQeII0gn37+k6Ru4Hng1DbEbWZmAxg0uUfEQ8AhfZSvpuh/ry9/FTi5JdGZmdmQ+A5VM7MMObmbmWXIyd3MLENO7mZmGXJyNzPLkJO7mVmGnNzNzDLk5G5mliEndzOzDDm5m5llyMndzCxDTu5mZhlycjczy5CTu5lZhpzczcwy1MhgHZXUtejWth/jycXHtf0YZmbt4DN3M7MMObmbmWXIyd3MLENO7mZmGXJyNzPLkJO7jWqSxkh6QNItaX5/SSskdUu6TtKOqXxcmu9Oy7vKjNtsME7uNtqdDTxWM38RsCQipgObgfmpfD6wOZUvSeuZVZaTu41akqYCxwHfTPMCZgM3pFWWASem6RPSPGn5nLS+WSWN2JuYrHo6cWMZtPTmsq8A5wK7pflJwJaI2Jbm1wJT0vQUYA1ARGyTtDWt/2z9TiUtABYA7Lvvvq2K1awpPnO3UUnS8cDGiLiv1fuOiKURMSsiZk2ePLnVuzdriM/cbbQ6EviIpLnAeGB34GJgoqSx6ex9KrAurb8OmAaslTQWmAA81/mwzRrjM3cblSLisxExNSK6gFOBOyPidOAu4KS02jzg5jS9PM2Tlt8ZEdHBkM2a4uRu1tt5wEJJ3RR96pen8suBSal8IbCopPjMGuJuGRv1IuJu4O40vRo4rI91XgVO7mhgZsPgM3czsww5uZuZZWjQ5C5pmqS7JK2S9Kiks1P5npJul/RE+rlHKpekS9Jt2g9JOrTdb8LMzHpr5Mx9G/CZiJgJHA6cJWkmxQWlOyJiBnAHb1xgOhaYkV4LgEtbHrWZmQ1o0OQeEesj4v40/SLFczim0Pt27PrbtK+Kwr0U3xvep+WRm5lZv5rqc09PwjsEWAHsHRHr06INwN5pevtt2kntLdxmZtYBDSd3SbsCNwLnRMQLtcvSzRxN3dAhaYGklZJWbtq0qZlNzcxsEA0ld0k7UCT2qyPiplT8TE93S/q5MZX33Kbdo/YW7u38/A0zs/Zp5Nsyorg777GI+PuaRbW3Y9ffpn1G+tbM4cDWmu4bMzPrgEbuUD0S+DjwsKQHU9n5wGLgeknzgaeAU9Ky24C5QDfwCnBmSyM2M7NBDZrcI+LHQH+DEszpY/0AzhpmXGZmNgy+Q9XMLENO7mZmGXJyNzPLkJO7mVmGnNzNzDLk5G5mliEndzOzDDm5m5llyMndzCxDHiDbzKwFuhbd2rJ9Pbn4uGHvw2fuZmYZcnI3M8uQk7uZWYac3M3MMuTkbmaWISd3M7MMObmbmWXIyd3MLENO7mZmGXJyNzPLkJO7mVmGnNzNzDLk5G5mliEndzOzDDm5m5llyMndzCxDTu5mZhlycrdRS9I0SXdJWiXpUUlnp/I9Jd0u6Yn0c49ULkmXSOqW9JCkQ8t9B2b9c3K30Wwb8JmImAkcDpwlaSawCLgjImYAd6R5gGOBGem1ALi08yGbNcbJ3UatiFgfEfen6ReBx4ApwAnAsrTaMuDENH0CcFUU7gUmStqnw2GbNcTJ3QyQ1AUcAqwA9o6I9WnRBmDvND0FWFOz2dpUVr+vBZJWSlq5adOmtsVsNhAndxv1JO0K3AicExEv1C6LiACimf1FxNKImBURsyZPntzCSM0aN2hyl3SFpI2SHqkp8wUny4KkHSgS+9URcVMqfqanuyX93JjK1wHTajafmsrMKqeRM/crgWPqynzByUY8SQIuBx6LiL+vWbQcmJem5wE315SfkU5iDge21nTfmFXKoMk9In4EPF9X7AtOloMjgY8DsyU9mF5zgcXAhyQ9ARyd5gFuA1YD3cBlwP8sIWazhowd4nbNXnDy2Y1VTkT8GFA/i+f0sX4AZ7U1KLMWGfYF1aFccAJ/o8DMrJ2GmtyHfcHJ3ygwM2ufoSZ3X3AyM6uwQfvcJV0LfBDYS9Ja4AKKC0zXS5oPPAWckla/DZhLccHpFeDMNsRsZmaDGDS5R8Rp/SzyBSczs4ryHapmZhlycjczy5CTu5lZhpzczcwy5ORuZpYhJ3czsww5uZuZZcjJ3cwsQ07uZmYZcnI3M8vQUJ/nbmaZ61p0a8v29eTi41q2L2uMz9zNzDLk5G5mliEndzOzDDm5m5llyMndzCxDTu5mZhlycjczy5CTu5lZhpzczcwy5ORuZpYhJ3czsww5uZuZZcgPDjMrmR/QZe3gM3czsww5uZuZZcjJ3cwsQ07uZmYZcnI3M8uQk7uZWYac3M3MMuTkbmaWISd3M7MMtSW5SzpG0uOSuiUtascxzMrgum0jRcuTu6QxwD8CxwIzgdMkzWz1ccw6zXXbRpJ2nLkfBnRHxOqIeA34NnBCG45j1mmu2zZitOPBYVOANTXza4H31a8kaQGwIM2+JOnxNsRSay/g2WY20EVtimT4/F6as1/TW/St7Lo96O+qpL+z42pOK+Pqt26X9lTIiFgKLO3U8SStjIhZnTpeO/m9VFu76nZVf1eOqzmdiqsd3TLrgGk181NTmdlI57ptI0Y7kvu/ATMk7S9pR+BUYHkbjmPWaa7bNmK0vFsmIrZJ+gvgX4AxwBUR8WirjzMEHesC6gC/lxJUoG5X9XfluJrTkbgUEZ04jpmZdZDvUDUzy5CTu5lZhpzczcwylGVylzRe0uQ+yidLGl9GTGbNSHX1TY82kDSzr7rdKW5bI0eWyR24BHh/H+VHAUs6HMuwSJoq6aia+YWSPpde08uMrVUkHSDpsrLjqJh/oLiTsd4k4OIOx1Krcm1rJLaRTtT5XJP7eyLipvrCiPgu8IES4hmOLwETa+Y/CbwMBPD5UiIaIknvkvQDSY9IulDSPpJuBO4EVpUdX8VMj4gf1RdGxD3Au0qIp0cV21Zl20iZdb60xw+02c4DLBtp/9DeERG31My/EhH/F0DSPSXFNFSXAZcCPwWOAR4ElgGnR8SrZQZWQbsNsGyHjkXxZlVsW1VuI6XV+ZGW6Bq1UdJh9YWS3gtsKiGe4ajvx5xTM93Xx/YqGxcRV0bE4xFxMfByRJzrxN6nbklz6wslHQusLiGeHlVsW1VuI6XV+VzP3P8KuF7SlcB9qWwWcAbFLeMjyYuSDoiIXwBExPMAkt4JvFhqZM0bL+kQQGn+N7XzEXF/aZFVzznArZJOoXcdPgI4vrSoqtm2qtxGSqvz2d6hKuktwFnAQanoEeAfI2JjeVE1T9IxFBexvgj0VIT3AOcDZ0fE98uKrVmS7qboB+1LRMTsDoZTeZLGAX/MG3X4UeCasj/pVK1tVbmNlFnns03utdJDng4E1o205A4g6SDgXIr3AEVj+lJEPFJeVNYJkvbnjb/7qogos0vmTarSttxG+hAR2b2ArwMHpukJFFelH6Z4POtpZcfXwve5b9kxNBnvuTXTJ9ct+7uy46vSC9gduB74JXAjcBNFX/t3gN1LjGtEta2y20iZdT7XC6rvjzee1ncm8IuI+C8UH9XOLS+soZF0hKST0sfhnq9XXQP8v5JDa1Ztn+xn65Yd08lARoBLKBLnjIj4WER8FHg7RSL9aolxVbJtVbiNlFbnc03ur9VMfwj4HkBEbCgnnKGT9CXgCuBjFBfYLgR+AKwAZpQZ2xCon+m+5ke7IyPibyLi9Z6CKPwtxUXVslSubVW8jZRW53P9tswWScdTfFQ8EpgPIGkssFOZgQ3BccAhEfGqpD0oxvA8KCKeLDesIYl+pvuat/6V+Y+wim2rym2ktDqfa3L/JMXH2rcC59ScVcwBbi0tqqF5NdK3IyJis6QnKlJph+Ldkl6gSE47pWnSvJ9L0ttPJH0O+EKkDloASX9NcUNMWarYtqrcRkqr86Pi2zIjmaQtQO1t6B9I86L4pP6RUgKztpK0O3A5cCjFXY0ABwMPAPMjYmtZsVWN20jfskzukv6BAT7yRMSnOxjOsEj6/TS5E0X/YQDdwK8BIuKHJYXWtPTUwE8B04GHKIap21ZuVNUm6e1Az9MhV0XELyWdExFfKSmeyrWtKreRMut8rsl93kDLI2JZp2IZLkk7UNyc8afA06l4GnAlcH5E/Lak0Jom6Trgt8A9wLHAUxFxdrlRjTySno6IfUs6duXaVpXbSJl1PsvknhNJS4BdgYUR8WIq2x34MsUDks4pM75mSHo4fW2u5wLczyLi0JLDGnEkrYmIaWXHURVVbiNl1vksL6hKWj7Q8hHWB3c8cEDtRbWIeEHSnwM/p3gGyUix/QwqIrZJ/vbjEJV2RlbRtlXlNlJanc8yuVN8D3gNcC3Fd11HchaJ6OPjVUT8TtJI+9jV880B6P3tgZ4LX7uXF1q1SHqRvpO4KPfrvFVsW1VuI6XV+VyT+1spbrA4jeLBS7cC19bcWTeSrJJ0RkRcVVso6U8ozkpGjIgYU3YMI0VEDPQ89zJVsW1Vto2UWeez73NPT9Y7jWK0ls9HRJm3bjdN0hSK54r8mt6PWN0J+O8Rsa6s2Gx0q0rbchvpW7bJPVW84ygqXxewnOJrSCPyDy1pNr2fDnhHmfHY6FXVtuU20luWyV3SVRTPmr4N+HaM5sd+mrWQ29bIkWtyf51igFzofVHKF+7MhsFta+TIMrmbmY12uT7y18xsVHNyNzPLkJO7mVmGnNzNzDLk5G5mlqH/D8/O/SuE1z5PAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lVCHjPhel4Kc"
      },
      "source": [
        "# Sources \n",
        "\n",
        "HuggingFace Documentation https://huggingface.co/docs/transformers/master/en/task_summary#named-entity-recognition"
      ]
    }
  ]
}