notebooks/huggingface_conll_2003.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Reproducing LUKE experimental results: CoNLL-2003",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMjXok147+LEzjGsmj66G9p",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "b37844d3b6604a30a1ff151537f2aa30": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_59509ddf1dac4a3a8644d4db7588b5fc",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0b12bbb032974cf99807a957830fb189",
              "IPY_MODEL_056befaa4afe437582f84cd567260f74"
            ]
          }
        },
        "59509ddf1dac4a3a8644d4db7588b5fc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0b12bbb032974cf99807a957830fb189": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a8c14e6e2d7245faaef43bd1dcfb9f43",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 877,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 877,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b55f83cc1abf44688d61d2fb9cc18173"
          }
        },
        "056befaa4afe437582f84cd567260f74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f506afb3d1f0419fa68b36ff2a4f4a54",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 877/877 [00:00&lt;00:00, 3.09kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_56dd814d5d7b40d4b3d2c286f2d6f79e"
          }
        },
        "a8c14e6e2d7245faaef43bd1dcfb9f43": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b55f83cc1abf44688d61d2fb9cc18173": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f506afb3d1f0419fa68b36ff2a4f4a54": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "56dd814d5d7b40d4b3d2c286f2d6f79e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6917109f47094803ab20d08cb366ee5b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_397dff83594143e39cecf60012da18c4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5dd10b7b5f574947beef21cbc953464f",
              "IPY_MODEL_8377f0e631cd42119c4908c21860d1eb"
            ]
          }
        },
        "397dff83594143e39cecf60012da18c4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5dd10b7b5f574947beef21cbc953464f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d725d9d7ffbe4a3a86b8fc02e2e1392b",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2239184431,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2239184431,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f268b328284a4bf6805152b0c9f40cd6"
          }
        },
        "8377f0e631cd42119c4908c21860d1eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ad5257d7f1894b26a0e6174b6d02d5aa",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2.24G/2.24G [01:08&lt;00:00, 32.7MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dfa085ecb98e435a858807784fd42b50"
          }
        },
        "d725d9d7ffbe4a3a86b8fc02e2e1392b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f268b328284a4bf6805152b0c9f40cd6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ad5257d7f1894b26a0e6174b6d02d5aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dfa085ecb98e435a858807784fd42b50": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bbfbd42d39e7429f8deab0cc2dd075b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8921b4a8756943848fa66823c2c5b44d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b46c4efa9fb04c24a38f63fdbff2651e",
              "IPY_MODEL_1a2d65ab95324e39b82804c3d87d33ba"
            ]
          }
        },
        "8921b4a8756943848fa66823c2c5b44d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b46c4efa9fb04c24a38f63fdbff2651e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_15ae8aa3e37f492aa2a06409602b9e39",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 898822,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 898822,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fb8baf778d0146a2af27a34d120bdb1b"
          }
        },
        "1a2d65ab95324e39b82804c3d87d33ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_77ad155fa70b40ec86c1e76b5c9bf5f0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 899k/899k [00:00&lt;00:00, 7.06MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c409450e949c4a3fb1046040ddfa28cc"
          }
        },
        "15ae8aa3e37f492aa2a06409602b9e39": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fb8baf778d0146a2af27a34d120bdb1b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "77ad155fa70b40ec86c1e76b5c9bf5f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c409450e949c4a3fb1046040ddfa28cc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f013f33c4d60452e8967cb2f67fa9826": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7444b17d20ff48f7afb28502e209e8cc",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_790936b91e0e4c1a9c85cad6b3230a69",
              "IPY_MODEL_abbe19061b2f42bab40c174120fc505f"
            ]
          }
        },
        "7444b17d20ff48f7afb28502e209e8cc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "790936b91e0e4c1a9c85cad6b3230a69": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c97bb09f1e1e4f22b1c26bbd03543ad4",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 456318,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 456318,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d54ddeb353fd4f69b6dbcdef5af10253"
          }
        },
        "abbe19061b2f42bab40c174120fc505f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f73f3b81ed7d4d94815a8df3969803e2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 456k/456k [00:01&lt;00:00, 436kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6729292e93414d01b79defa7c5e12a69"
          }
        },
        "c97bb09f1e1e4f22b1c26bbd03543ad4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d54ddeb353fd4f69b6dbcdef5af10253": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f73f3b81ed7d4d94815a8df3969803e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6729292e93414d01b79defa7c5e12a69": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6c9aaf14fd1045969b885707192fdeb2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d2711f188d0c4bc795cba6a5a0f9a278",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7dcfd76ad36b44eaaad219168b541ec2",
              "IPY_MODEL_2bb01d3b5cf04e0cbe2568b4c331e534"
            ]
          }
        },
        "d2711f188d0c4bc795cba6a5a0f9a278": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7dcfd76ad36b44eaaad219168b541ec2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_57ad8d00f6674dad83b60843a5e0b105",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 15287192,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 15287192,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6a48baa290b047a0877af3924fa57af7"
          }
        },
        "2bb01d3b5cf04e0cbe2568b4c331e534": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_150d2e06f60b4b2090b02f22ec9932c0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 15.3M/15.3M [00:00&lt;00:00, 32.3MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d04446ecbb7447ed9cb41e61dc59bade"
          }
        },
        "57ad8d00f6674dad83b60843a5e0b105": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6a48baa290b047a0877af3924fa57af7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "150d2e06f60b4b2090b02f22ec9932c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d04446ecbb7447ed9cb41e61dc59bade": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "633d91deb7bf493e996027e41609739a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_97ad753bff6b40a3b02b50c997290716",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3ea8238444004aab80c84378adec959c",
              "IPY_MODEL_905999786dd7463baf98bf70c64a3c44"
            ]
          }
        },
        "97ad753bff6b40a3b02b50c997290716": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3ea8238444004aab80c84378adec959c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7caf19d339444549a3d933fb9aee7054",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 33,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 33,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_38b97d52ee5d4ea49a799ad8fe4794f5"
          }
        },
        "905999786dd7463baf98bf70c64a3c44": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3b4ee50defb8471997d90d13ef531e3b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 33.0/33.0 [00:00&lt;00:00, 330B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c611c76ff8894b0a91f2762ea9ba4bad"
          }
        },
        "7caf19d339444549a3d933fb9aee7054": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "38b97d52ee5d4ea49a799ad8fe4794f5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3b4ee50defb8471997d90d13ef531e3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c611c76ff8894b0a91f2762ea9ba4bad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "70da79ac96e4412b9612883ab62ea79c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_07775668d7d3494e97579403766fa0ef",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_39c7bf2eaeb54fc6b3761c036ad7ef1b",
              "IPY_MODEL_9fd9f33c35454fe6be50f5fb32a895d5"
            ]
          }
        },
        "07775668d7d3494e97579403766fa0ef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "39c7bf2eaeb54fc6b3761c036ad7ef1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a5169cb0a8af432691556c7fad45a436",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1000,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ba4352aa2f9140e99f7facf56d608d0c"
          }
        },
        "9fd9f33c35454fe6be50f5fb32a895d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5b58c57efbce4bb4b3e1fadd9ec65d7e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.00k/1.00k [00:00&lt;00:00, 4.10kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8bebbe73c86e4443a3c5051a79bf86cd"
          }
        },
        "a5169cb0a8af432691556c7fad45a436": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ba4352aa2f9140e99f7facf56d608d0c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5b58c57efbce4bb4b3e1fadd9ec65d7e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8bebbe73c86e4443a3c5051a79bf86cd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e776b6c1b81744e3b8d17c2fa92bb337": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_57403b214cb44923b6a1a728adb9c1c1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c7def66eec6940b7a901888a0f262a02",
              "IPY_MODEL_d28faf38aae046c98eada52d7f03a2e2"
            ]
          }
        },
        "57403b214cb44923b6a1a728adb9c1c1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c7def66eec6940b7a901888a0f262a02": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_96779da8c0824402b0f5adc0a6f4707a",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1695,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1695,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_46726698bf754d7c8d7e57b4c4c7f5da"
          }
        },
        "d28faf38aae046c98eada52d7f03a2e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_650425db2f50495d882c04b1cd414607",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.70k/1.70k [00:00&lt;00:00, 11.8kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_04e9a0256e774a1d837dfc0a470049c4"
          }
        },
        "96779da8c0824402b0f5adc0a6f4707a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "46726698bf754d7c8d7e57b4c4c7f5da": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "650425db2f50495d882c04b1cd414607": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "04e9a0256e774a1d837dfc0a470049c4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/studio-ousia/luke/blob/master/notebooks/huggingface_conll_2003.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tRs4Wyghwse7"
      },
      "source": [
        "# Reproducing experimental results of LUKE on CoNLL-2003 Using Hugging Face Transformers\n",
        "\n",
        "This notebook shows how to reproduce the state-of-the-art results on the [CoNLL-2003 named entity recognition dataset](https://www.clips.uantwerpen.be/conll2003/ner/) reported in [this paper](https://arxiv.org/abs/2010.01057) using the Trasnsformers library and the [fine-tuned model checkpoint](https://huggingface.co/studio-ousia/luke-large-finetuned-conll-2003) available on the Model Hub.\n",
        "The source code used in the experiments is also available [here](https://github.com/studio-ousia/luke/tree/master/examples/ner).\n",
        "\n",
        "*Currently, due to the slight difference in preprocessing, the score reproduced in this notebook is slightly lower than the score reported in the original paper (approximately 0.1 F1).*\n",
        "\n",
        "There are two other related notebooks:\n",
        "\n",
        "* [Reproducing experimental results of LUKE on Open Entity Using Hugging Face Transformers](https://github.com/studio-ousia/luke/blob/master/notebooks/huggingface_open_entity.ipynb)\n",
        "* [Reproducing experimental results of LUKE on TACRED Using Hugging Face Transformers](https://github.com/studio-ousia/luke/blob/master/notebooks/huggingface_tacred.ipynb)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "18lGoU-SmTK8",
        "outputId": "960c163c-5aa7-4b16-8cbd-16a07891c973"
      },
      "source": [
        "# Currently, LUKE is only available on the master branch\n",
        "!pip install seqeval git+https://github.com/huggingface/transformers.git"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting git+https://github.com/huggingface/transformers.git\n",
            "  Cloning https://github.com/huggingface/transformers.git to /tmp/pip-req-build-v2tzy73s\n",
            "  Running command git clone -q https://github.com/huggingface/transformers.git /tmp/pip-req-build-v2tzy73s\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting seqeval\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9d/2d/233c79d5b4e5ab1dbf111242299153f3caddddbb691219f363ad55ce783d/seqeval-1.2.2.tar.gz (43kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 7.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.6.0.dev0) (2019.12.20)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.6.0.dev0) (1.19.5)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers==4.6.0.dev0) (3.10.1)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n",
            "\u001b[K     |████████████████████████████████| 901kB 26.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers==4.6.0.dev0) (4.41.1)\n",
            "Collecting huggingface-hub==0.0.8\n",
            "  Downloading https://files.pythonhosted.org/packages/a1/88/7b1e45720ecf59c6c6737ff332f41c955963090a18e72acbcbeac6b25e86/huggingface_hub-0.0.8-py3-none-any.whl\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers==4.6.0.dev0) (3.0.12)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ae/04/5b870f26a858552025a62f1649c20d29d2672c02ff3c3fb4c688ca46467a/tokenizers-0.10.2-cp37-cp37m-manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 50.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers==4.6.0.dev0) (2.23.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers==4.6.0.dev0) (20.9)\n",
            "Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.7/dist-packages (from seqeval) (0.22.2.post1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers==4.6.0.dev0) (3.4.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers==4.6.0.dev0) (3.7.4.3)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.6.0.dev0) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.6.0.dev0) (1.0.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.6.0.dev0) (1.15.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.6.0.dev0) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.6.0.dev0) (2020.12.5)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.6.0.dev0) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.6.0.dev0) (1.24.3)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers==4.6.0.dev0) (2.4.7)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.4.1)\n",
            "Building wheels for collected packages: transformers\n",
            "  Building wheel for transformers (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for transformers: filename=transformers-4.6.0.dev0-cp37-none-any.whl size=2168830 sha256=850e2b7ddd249e04daaa0a59fd9662d95a883d7b136002b19f78fce76a04f402\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-hgg1rs92/wheels/33/eb/3b/4bf5dd835e865e472d4fc0754f35ac0edb08fe852e8f21655f\n",
            "Successfully built transformers\n",
            "Building wheels for collected packages: seqeval\n",
            "  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for seqeval: filename=seqeval-1.2.2-cp37-none-any.whl size=16172 sha256=bb79d0002e7f87a0c0d0134ae067015a0de1d8993895303282fc78c6bc93cb0e\n",
            "  Stored in directory: /root/.cache/pip/wheels/52/df/1b/45d75646c37428f7e626214704a0e35bd3cfc32eda37e59e5f\n",
            "Successfully built seqeval\n",
            "Installing collected packages: seqeval, sacremoses, huggingface-hub, tokenizers, transformers\n",
            "Successfully installed huggingface-hub-0.0.8 sacremoses-0.0.45 seqeval-1.2.2 tokenizers-0.10.2 transformers-4.6.0.dev0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RBERtWDCnqXO"
      },
      "source": [
        "import unicodedata\n",
        "\n",
        "import numpy as np\n",
        "import seqeval.metrics\n",
        "import spacy\n",
        "import torch\n",
        "from tqdm import tqdm, trange\n",
        "from transformers import LukeTokenizer, LukeForEntitySpanClassification"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yYBJqG6JBNes"
      },
      "source": [
        "## Loading the dataset\n",
        "\n",
        "The test set of the CoNLL-2003 dataset (eng.testb) is placed in the current directory and loaded using `load_examples` function."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tLzX8LIS127b",
        "outputId": "125b5b66-0aba-4284-ea53-3178b64a0c5d"
      },
      "source": [
        "# Download the testb set of the CoNLL-2003 dataset\n",
        "!wget https://raw.githubusercontent.com/synalp/NER/master/corpus/CoNLL-2003/eng.testb"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-05-04 04:15:54--  https://raw.githubusercontent.com/synalp/NER/master/corpus/CoNLL-2003/eng.testb\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.110.133, 185.199.109.133, 185.199.111.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.110.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 748096 (731K) [text/plain]\n",
            "Saving to: ‘eng.testb’\n",
            "\n",
            "\reng.testb             0%[                    ]       0  --.-KB/s               \reng.testb           100%[===================>] 730.56K  --.-KB/s    in 0.009s  \n",
            "\n",
            "2021-05-04 04:15:54 (82.8 MB/s) - ‘eng.testb’ saved [748096/748096]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3gfWDJsgBjGw"
      },
      "source": [
        "## Loading the fine-tuned model and tokenizer\n",
        "\n",
        "We construct the model and tokenizer using the [fine-tuned model checkpoint](https://huggingface.co/studio-ousia/luke-large-finetuned-conll-2003)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 480,
          "referenced_widgets": [
            "b37844d3b6604a30a1ff151537f2aa30",
            "59509ddf1dac4a3a8644d4db7588b5fc",
            "0b12bbb032974cf99807a957830fb189",
            "056befaa4afe437582f84cd567260f74",
            "a8c14e6e2d7245faaef43bd1dcfb9f43",
            "b55f83cc1abf44688d61d2fb9cc18173",
            "f506afb3d1f0419fa68b36ff2a4f4a54",
            "56dd814d5d7b40d4b3d2c286f2d6f79e",
            "6917109f47094803ab20d08cb366ee5b",
            "397dff83594143e39cecf60012da18c4",
            "5dd10b7b5f574947beef21cbc953464f",
            "8377f0e631cd42119c4908c21860d1eb",
            "d725d9d7ffbe4a3a86b8fc02e2e1392b",
            "f268b328284a4bf6805152b0c9f40cd6",
            "ad5257d7f1894b26a0e6174b6d02d5aa",
            "dfa085ecb98e435a858807784fd42b50",
            "bbfbd42d39e7429f8deab0cc2dd075b6",
            "8921b4a8756943848fa66823c2c5b44d",
            "b46c4efa9fb04c24a38f63fdbff2651e",
            "1a2d65ab95324e39b82804c3d87d33ba",
            "15ae8aa3e37f492aa2a06409602b9e39",
            "fb8baf778d0146a2af27a34d120bdb1b",
            "77ad155fa70b40ec86c1e76b5c9bf5f0",
            "c409450e949c4a3fb1046040ddfa28cc",
            "f013f33c4d60452e8967cb2f67fa9826",
            "7444b17d20ff48f7afb28502e209e8cc",
            "790936b91e0e4c1a9c85cad6b3230a69",
            "abbe19061b2f42bab40c174120fc505f",
            "c97bb09f1e1e4f22b1c26bbd03543ad4",
            "d54ddeb353fd4f69b6dbcdef5af10253",
            "f73f3b81ed7d4d94815a8df3969803e2",
            "6729292e93414d01b79defa7c5e12a69",
            "6c9aaf14fd1045969b885707192fdeb2",
            "d2711f188d0c4bc795cba6a5a0f9a278",
            "7dcfd76ad36b44eaaad219168b541ec2",
            "2bb01d3b5cf04e0cbe2568b4c331e534",
            "57ad8d00f6674dad83b60843a5e0b105",
            "6a48baa290b047a0877af3924fa57af7",
            "150d2e06f60b4b2090b02f22ec9932c0",
            "d04446ecbb7447ed9cb41e61dc59bade",
            "633d91deb7bf493e996027e41609739a",
            "97ad753bff6b40a3b02b50c997290716",
            "3ea8238444004aab80c84378adec959c",
            "905999786dd7463baf98bf70c64a3c44",
            "7caf19d339444549a3d933fb9aee7054",
            "38b97d52ee5d4ea49a799ad8fe4794f5",
            "3b4ee50defb8471997d90d13ef531e3b",
            "c611c76ff8894b0a91f2762ea9ba4bad",
            "70da79ac96e4412b9612883ab62ea79c",
            "07775668d7d3494e97579403766fa0ef",
            "39c7bf2eaeb54fc6b3761c036ad7ef1b",
            "9fd9f33c35454fe6be50f5fb32a895d5",
            "a5169cb0a8af432691556c7fad45a436",
            "ba4352aa2f9140e99f7facf56d608d0c",
            "5b58c57efbce4bb4b3e1fadd9ec65d7e",
            "8bebbe73c86e4443a3c5051a79bf86cd",
            "e776b6c1b81744e3b8d17c2fa92bb337",
            "57403b214cb44923b6a1a728adb9c1c1",
            "c7def66eec6940b7a901888a0f262a02",
            "d28faf38aae046c98eada52d7f03a2e2",
            "96779da8c0824402b0f5adc0a6f4707a",
            "46726698bf754d7c8d7e57b4c4c7f5da",
            "650425db2f50495d882c04b1cd414607",
            "04e9a0256e774a1d837dfc0a470049c4"
          ]
        },
        "id": "q9bXAEPZp0ZT",
        "outputId": "9ff207ee-4339-4459-8e61-702299b8c317"
      },
      "source": [
        "# Load the model checkpoint\n",
        "model = LukeForEntitySpanClassification.from_pretrained(\"studio-ousia/luke-large-finetuned-conll-2003\")\n",
        "model.eval()\n",
        "model.to(\"cuda\")\n",
        "\n",
        "# Load the tokenizer\n",
        "tokenizer = LukeTokenizer.from_pretrained(\"studio-ousia/luke-large-finetuned-conll-2003\")"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b37844d3b6604a30a1ff151537f2aa30",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=877.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6917109f47094803ab20d08cb366ee5b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=2239184431.0, style=ProgressStyle(descr…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at studio-ousia/luke-large-finetuned-conll-2003 were not used when initializing LukeForEntitySpanClassification: ['luke.embeddings.position_ids']\n",
            "- This IS expected if you are initializing LukeForEntitySpanClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing LukeForEntitySpanClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bbfbd42d39e7429f8deab0cc2dd075b6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=898822.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f013f33c4d60452e8967cb2f67fa9826",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=456318.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6c9aaf14fd1045969b885707192fdeb2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=15287192.0, style=ProgressStyle(descrip…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "633d91deb7bf493e996027e41609739a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=33.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "70da79ac96e4412b9612883ab62ea79c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1000.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e776b6c1b81744e3b8d17c2fa92bb337",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1695.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zz-nZAmVBopz"
      },
      "source": [
        "## Loading the dataset\n",
        "\n",
        "First, the documents in the dataset is loaded using the `load_documents` function. This function outputs the list of dicts which have the following three keys:\n",
        "* `words`: the sequence of words\n",
        "* `labels`: the sequence of gold-standard NER labels (`\"MISC\"`, `\"PER\"`, `\"ORG\"`, or `\"LOC\"`)\n",
        "* `sentence_boundaries`: positions of sentence boundaries in the word sequence\n",
        "\n",
        "The `load_examples` function creates a batch instance for each sentence in a document.\n",
        "The model addresses the task by classifying all possible entity spans in a sentence into `[\"NIL\", \"MISC\", \"PER\", \"ORG\", \"LOC\"]`, where `\"NIL\"` represents that the span is not an entity name (see Section 4.3 in the [original paper](https://arxiv.org/abs/2010.01057)).\n",
        "Here, we create the list of all possible entity spans (character-based start and entity positions) in a sentence.\n",
        "Specifically, this function returns the list of dicts with the following four keys:\n",
        "* `text`: text\n",
        "* `words`: the sequence of words\n",
        "* `entity_spans`: the list of possible entity spans (character-based start and end positions in the `text`)\n",
        "* `original_word_spans`: the list of corresponding spans of `entity_spans` in the word sequence\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y-v01ix-i2FS"
      },
      "source": [
        "def load_documents(dataset_file):\n",
        "    documents = []\n",
        "    words = []\n",
        "    labels = []\n",
        "    sentence_boundaries = []\n",
        "    with open(dataset_file) as f:\n",
        "        for line in f:\n",
        "            line = line.rstrip()\n",
        "            if line.startswith(\"-DOCSTART\"):\n",
        "                if words:\n",
        "                    documents.append(dict(\n",
        "                        words=words,\n",
        "                        labels=labels,\n",
        "                        sentence_boundaries=sentence_boundaries\n",
        "                    ))\n",
        "                    words = []\n",
        "                    labels = []\n",
        "                    sentence_boundaries = []\n",
        "                continue\n",
        "\n",
        "            if not line:\n",
        "                if not sentence_boundaries or len(words) != sentence_boundaries[-1]:\n",
        "                    sentence_boundaries.append(len(words))\n",
        "            else:\n",
        "                items = line.split(\" \")\n",
        "                words.append(items[0])\n",
        "                labels.append(items[-1])\n",
        "\n",
        "    if words:\n",
        "        documents.append(dict(\n",
        "            words=words,\n",
        "            labels=labels,\n",
        "            sentence_boundaries=sentence_boundaries\n",
        "        ))\n",
        "        \n",
        "    return documents\n",
        "\n",
        "\n",
        "def load_examples(documents):\n",
        "    examples = []\n",
        "    max_token_length = 510\n",
        "    max_mention_length = 30\n",
        "\n",
        "    for document in tqdm(documents):\n",
        "        words = document[\"words\"]\n",
        "        subword_lengths = [len(tokenizer.tokenize(w)) for w in words]\n",
        "        total_subword_length = sum(subword_lengths)\n",
        "        sentence_boundaries = document[\"sentence_boundaries\"]\n",
        "\n",
        "        for i in range(len(sentence_boundaries) - 1):\n",
        "            sentence_start, sentence_end = sentence_boundaries[i:i+2]\n",
        "            if total_subword_length <= max_token_length:\n",
        "                # if the total sequence length of the document is shorter than the\n",
        "                # maximum token length, we simply use all words to build the sequence\n",
        "                context_start = 0\n",
        "                context_end = len(words)\n",
        "            else:\n",
        "                # if the total sequence length is longer than the maximum length, we add\n",
        "                # the surrounding words of the target sentence　to the sequence until it\n",
        "                # reaches the maximum length\n",
        "                context_start = sentence_start\n",
        "                context_end = sentence_end\n",
        "                cur_length = sum(subword_lengths[context_start:context_end])\n",
        "                while True:\n",
        "                    if context_start > 0:\n",
        "                        if cur_length + subword_lengths[context_start - 1] <= max_token_length:\n",
        "                            cur_length += subword_lengths[context_start - 1]\n",
        "                            context_start -= 1\n",
        "                        else:\n",
        "                            break\n",
        "                    if context_end < len(words):\n",
        "                        if cur_length + subword_lengths[context_end] <= max_token_length:\n",
        "                            cur_length += subword_lengths[context_end]\n",
        "                            context_end += 1\n",
        "                        else:\n",
        "                            break\n",
        "\n",
        "            text = \"\"\n",
        "            for word in words[context_start:sentence_start]:\n",
        "                if word[0] == \"'\" or (len(word) == 1 and is_punctuation(word)):\n",
        "                    text = text.rstrip()\n",
        "                text += word\n",
        "                text += \" \"\n",
        "\n",
        "            sentence_words = words[sentence_start:sentence_end]\n",
        "            sentence_subword_lengths = subword_lengths[sentence_start:sentence_end]\n",
        "\n",
        "            word_start_char_positions = []\n",
        "            word_end_char_positions = []\n",
        "            for word in sentence_words:\n",
        "                if word[0] == \"'\" or (len(word) == 1 and is_punctuation(word)):\n",
        "                    text = text.rstrip()\n",
        "                word_start_char_positions.append(len(text))\n",
        "                text += word\n",
        "                word_end_char_positions.append(len(text))\n",
        "                text += \" \"\n",
        "\n",
        "            for word in words[sentence_end:context_end]:\n",
        "                if word[0] == \"'\" or (len(word) == 1 and is_punctuation(word)):\n",
        "                    text = text.rstrip()\n",
        "                text += word\n",
        "                text += \" \"\n",
        "            text = text.rstrip()\n",
        "\n",
        "            entity_spans = []\n",
        "            original_word_spans = []\n",
        "            for word_start in range(len(sentence_words)):\n",
        "                for word_end in range(word_start, len(sentence_words)):\n",
        "                    if sum(sentence_subword_lengths[word_start:word_end + 1]) <= max_mention_length:\n",
        "                        entity_spans.append(\n",
        "                            (word_start_char_positions[word_start], word_end_char_positions[word_end])\n",
        "                        )\n",
        "                        original_word_spans.append(\n",
        "                            (word_start, word_end + 1)\n",
        "                        )\n",
        "\n",
        "            examples.append(dict(\n",
        "                text=text,\n",
        "                words=sentence_words,\n",
        "                entity_spans=entity_spans,\n",
        "                original_word_spans=original_word_spans,\n",
        "            ))\n",
        "\n",
        "    return examples\n",
        "\n",
        "\n",
        "def is_punctuation(char):\n",
        "    cp = ord(char)\n",
        "    if (cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126):\n",
        "        return True\n",
        "    cat = unicodedata.category(char)\n",
        "    if cat.startswith(\"P\"):\n",
        "        return True\n",
        "    return False"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_H6us8Unl231",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ccaf4e22-eb71-44c5-a903-660ebf0c195a"
      },
      "source": [
        "test_documents = load_documents(\"eng.testb\")\n",
        "test_examples = load_examples(test_documents)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 231/231 [00:04<00:00, 49.40it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eomnOqQKDNUX"
      },
      "source": [
        "## Measuring performance\n",
        "\n",
        "We classify all possible entity spans in the test set, exclude all spans classified into the `NIL` type, and greedily select a span from the remaining spans based on the logit of its predicted entity type in descending order.\n",
        "Due to  minor differences in processing, the reproduced performance is slightly lower than the performance reported in the [original paper](https://arxiv.org/abs/2010.01057) (approximately 0.1 F1)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uGnyU7rGnNg2",
        "outputId": "3338eb50-bd8e-40ac-913e-a6a3366796a9"
      },
      "source": [
        "batch_size = 2\n",
        "all_logits = []\n",
        "\n",
        "for batch_start_idx in trange(0, len(test_examples), batch_size):\n",
        "    batch_examples = test_examples[batch_start_idx:batch_start_idx + batch_size]\n",
        "    texts = [example[\"text\"] for example in batch_examples]\n",
        "    entity_spans = [example[\"entity_spans\"] for example in batch_examples]\n",
        "\n",
        "    inputs = tokenizer(texts, entity_spans=entity_spans, return_tensors=\"pt\", padding=True)\n",
        "    inputs = inputs.to(\"cuda\")\n",
        "    with torch.no_grad():\n",
        "        outputs = model(**inputs)\n",
        "    all_logits.extend(outputs.logits.tolist())"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 1727/1727 [09:35<00:00,  3.00it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FcNmRnjne339"
      },
      "source": [
        "final_labels = [label for document in test_documents for label in document[\"labels\"]]\n",
        "\n",
        "final_predictions = []\n",
        "for example_index, example in enumerate(test_examples):\n",
        "    logits = all_logits[example_index]\n",
        "    max_logits = np.max(logits, axis=1)\n",
        "    max_indices = np.argmax(logits, axis=1)\n",
        "    original_spans = example[\"original_word_spans\"]\n",
        "    predictions = []\n",
        "    for logit, index, span in zip(max_logits, max_indices, original_spans):\n",
        "        if index != 0:  # the span is not NIL\n",
        "            predictions.append((logit, span, model.config.id2label[index]))\n",
        "\n",
        "    # construct an IOB2 label sequence\n",
        "    predicted_sequence = [\"O\"] * len(example[\"words\"])\n",
        "    for _, span, label in sorted(predictions, key=lambda o: o[0], reverse=True):\n",
        "        if all([o == \"O\" for o in predicted_sequence[span[0] : span[1]]]):\n",
        "            predicted_sequence[span[0]] = \"B-\" + label\n",
        "            if span[1] - span[0] > 1:\n",
        "                predicted_sequence[span[0] + 1 : span[1]] = [\"I-\" + label] * (span[1] - span[0] - 1)\n",
        "\n",
        "    final_predictions += predicted_sequence"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yFEYKr9tuk36",
        "outputId": "3f68e281-2475-4dba-8a31-9f88a82e2835"
      },
      "source": [
        "print(seqeval.metrics.classification_report([final_labels], [final_predictions], digits=4)) "
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "         LOC     0.9558    0.9478    0.9518      1666\n",
            "        MISC     0.8553    0.8688    0.8620       701\n",
            "         ORG     0.9287    0.9496    0.9391      1647\n",
            "         PER     0.9683    0.9719    0.9701      1602\n",
            "\n",
            "   micro avg     0.9386    0.9453    0.9420      5616\n",
            "   macro avg     0.9270    0.9345    0.9307      5616\n",
            "weighted avg     0.9389    0.9453    0.9421      5616\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TtMnpKyGATY-"
      },
      "source": [
        "## Recognizing named entities in a text\n",
        "\n",
        "Finally, we extract named entities from a text using the [fine-tuned model](https://huggingface.co/studio-ousia/luke-large-finetuned-conll-2003). The input text is tokenized using [SpaCy](https://spacy.io/)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L3U75A-27yTj",
        "outputId": "99f5fbc4-0990-401f-e11f-c094e070b49e"
      },
      "source": [
        "text = \"Star Wars is a film written and directed by George Lucas\"\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "doc = nlp(text)\n",
        "\n",
        "entity_spans = []\n",
        "original_word_spans = []\n",
        "for token_start in doc:\n",
        "    for token_end in doc[token_start.i:]:\n",
        "        entity_spans.append((token_start.idx, token_end.idx + len(token_end)))\n",
        "        original_word_spans.append((token_start.i, token_end.i + 1))\n",
        "\n",
        "inputs = tokenizer(text, entity_spans=entity_spans, return_tensors=\"pt\", padding=True)\n",
        "inputs = inputs.to(\"cuda\")\n",
        "with torch.no_grad():\n",
        "    outputs = model(**inputs)\n",
        "\n",
        "logits = outputs.logits\n",
        "max_logits, max_indices = logits[0].max(dim=1)\n",
        "\n",
        "predictions = []\n",
        "for logit, index, span in zip(max_logits, max_indices, original_word_spans):\n",
        "    if index != 0:  # the span is not NIL\n",
        "        predictions.append((logit, span, model.config.id2label[int(index)]))\n",
        "\n",
        "# construct an IOB2 label sequence\n",
        "predicted_sequence = [\"O\"] * len(doc)\n",
        "for _, span, label in sorted(predictions, key=lambda o: o[0], reverse=True):\n",
        "    if all([o == \"O\" for o in predicted_sequence[span[0] : span[1]]]):\n",
        "        predicted_sequence[span[0]] = \"B-\" + label\n",
        "        if span[1] - span[0] > 1:\n",
        "            predicted_sequence[span[0] + 1 : span[1]] = [\"I-\" + label] * (span[1] - span[0] - 1)\n",
        "\n",
        "for token, label in zip(doc, predicted_sequence):\n",
        "    print(token, label)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Star B-MISC\n",
            "Wars I-MISC\n",
            "is O\n",
            "a O\n",
            "film O\n",
            "written O\n",
            "and O\n",
            "directed O\n",
            "by O\n",
            "George B-PER\n",
            "Lucas I-PER\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}