xlnet.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "xlnet.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "TPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "ab5a0f39b374453f97d65f2cb6454c07": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_dc0fe31513954da5aa1708e7230c18ec",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_62c4854a1305487f998e4ba0d65b9e91",
              "IPY_MODEL_38aa137682af4c78b4c3e140405b65c3",
              "IPY_MODEL_607af8ca61134e4db9aa6aa6828b3d76"
            ]
          }
        },
        "dc0fe31513954da5aa1708e7230c18ec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "62c4854a1305487f998e4ba0d65b9e91": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f59babf93dc34608843d7a078a3d717c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_62d7243a3113495a94605c5217331b34"
          }
        },
        "38aa137682af4c78b4c3e140405b65c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_87c23e85ece748faa765c1a669e06a9d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 798011,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 798011,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4be046a6ef4f4b7999cfc926fb93d2b9"
          }
        },
        "607af8ca61134e4db9aa6aa6828b3d76": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_40d1000a9fef4045aa34136f967ecc80",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 779k/779k [00:00&lt;00:00, 2.05MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_eec03b58bac7479fb2677e79ae57d252"
          }
        },
        "f59babf93dc34608843d7a078a3d717c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "62d7243a3113495a94605c5217331b34": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "87c23e85ece748faa765c1a669e06a9d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4be046a6ef4f4b7999cfc926fb93d2b9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "40d1000a9fef4045aa34136f967ecc80": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "eec03b58bac7479fb2677e79ae57d252": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "da9e888309fd465d8cdbf022af128296": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0b952341bbde46fca5970f470d10c6d3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b8f9d9ce77454b5eb96d5b1a8b033b66",
              "IPY_MODEL_b620d458522c47fd8494c52bf46aa8db",
              "IPY_MODEL_c6f992f372974692b20b293fb5a2ca8d"
            ]
          }
        },
        "0b952341bbde46fca5970f470d10c6d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b8f9d9ce77454b5eb96d5b1a8b033b66": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b5e463792f9940df8979550fc645a5c4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_113539e35b0d4756a5dc866503ab1601"
          }
        },
        "b620d458522c47fd8494c52bf46aa8db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6b7f508035f1460d8b3398caeeacc8f0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1382015,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1382015,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_acecf30fb58246f68927c94a31258022"
          }
        },
        "c6f992f372974692b20b293fb5a2ca8d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9faae80009cb483db0134cb2f8f7d54a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.32M/1.32M [00:00&lt;00:00, 1.38MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ed7dd3b49e8e48fc875712c561460d66"
          }
        },
        "b5e463792f9940df8979550fc645a5c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "113539e35b0d4756a5dc866503ab1601": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6b7f508035f1460d8b3398caeeacc8f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "acecf30fb58246f68927c94a31258022": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9faae80009cb483db0134cb2f8f7d54a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ed7dd3b49e8e48fc875712c561460d66": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "85ed8e75a5de4676a57fdc57a19a2f78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_16d78c976f564839b1e23b07a10951ac",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9370800b70c0453e85603c3dbc2a9209",
              "IPY_MODEL_7394e1276fd54769a0f69501adf64b15",
              "IPY_MODEL_59e002e2678846b087fa240a8d129435"
            ]
          }
        },
        "16d78c976f564839b1e23b07a10951ac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9370800b70c0453e85603c3dbc2a9209": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d0606f59320542429a7e39d8e9bf8ba5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_199abf1bb30541c8b4ba7b1ab4b618bb"
          }
        },
        "7394e1276fd54769a0f69501adf64b15": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_32786184f6d84f37bd1b040113957692",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 761,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 761,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1f2a05bce0f7464d94f3cd5cf459afc1"
          }
        },
        "59e002e2678846b087fa240a8d129435": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_77a0e785fa7c4f19b6f517c2d13705d1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 761/761 [00:00&lt;00:00, 16.2kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_70c2952ddc2245b48906769adef31fcf"
          }
        },
        "d0606f59320542429a7e39d8e9bf8ba5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "199abf1bb30541c8b4ba7b1ab4b618bb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "32786184f6d84f37bd1b040113957692": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1f2a05bce0f7464d94f3cd5cf459afc1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "77a0e785fa7c4f19b6f517c2d13705d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "70c2952ddc2245b48906769adef31fcf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "da9eca3ff8ea4f01969ede286fc67dab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f68454de03884ae588c3ec5250c14a80",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9cb2793b151a4461acc875b8943a5230",
              "IPY_MODEL_1e961f27ec624f6aa37b2d44b50c53b6",
              "IPY_MODEL_8a179d20736e471795d36e8d87e654c6"
            ]
          }
        },
        "f68454de03884ae588c3ec5250c14a80": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9cb2793b151a4461acc875b8943a5230": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_eb4cdbb857644dc5a1abb32a41e7d9ad",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_68d4850031c8495abbccb0ab1cb96fb8"
          }
        },
        "1e961f27ec624f6aa37b2d44b50c53b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_fad48accdbda4ef3875c84a0e1e0222d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1572621064,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1572621064,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d045688efd784f67b9a3d7e64e047ba5"
          }
        },
        "8a179d20736e471795d36e8d87e654c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f3ff1041d8ea4c8ba3abff13bb6b3e1c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.46G/1.46G [00:36&lt;00:00, 35.8MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_50edf4507d5c455eade09c0a13d31dc2"
          }
        },
        "eb4cdbb857644dc5a1abb32a41e7d9ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "68d4850031c8495abbccb0ab1cb96fb8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fad48accdbda4ef3875c84a0e1e0222d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d045688efd784f67b9a3d7e64e047ba5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f3ff1041d8ea4c8ba3abff13bb6b3e1c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "50edf4507d5c455eade09c0a13d31dc2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_hYuEAOxSlvf"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "df=pd.read_csv(\"/content/FINAL_DATA2.csv\")\n",
        "df.cleanData=df.cleanData.astype(str)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install sentencepiece transformers\n",
        "\n",
        "import sentencepiece\n",
        "from transformers import XLNetTokenizer, TFXLNetModel,XLNetModel"
      ],
      "metadata": {
        "id": "hhWpooHNUV-y",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b6391f15-3121-4633-f7a8-b56ca7156b47"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting sentencepiece\n",
            "  Downloading sentencepiece-0.1.96-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2 MB 5.0 MB/s \n",
            "\u001b[?25hCollecting transformers\n",
            "  Downloading transformers-4.15.0-py3-none-any.whl (3.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.4 MB 29.5 MB/s \n",
            "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n",
            "  Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3 MB 26.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.47-py2.py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 57.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.10.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.4.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.62.3)\n",
            "Collecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.4.0-py3-none-any.whl (67 kB)\n",
            "\u001b[K     |████████████████████████████████| 67 kB 4.7 MB/s \n",
            "\u001b[?25hCollecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 61.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.6)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.7.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.10.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.1.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Installing collected packages: pyyaml, tokenizers, sacremoses, huggingface-hub, transformers, sentencepiece\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed huggingface-hub-0.4.0 pyyaml-6.0 sacremoses-0.0.47 sentencepiece-0.1.96 tokenizers-0.10.3 transformers-4.15.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "import seaborn as sns\n",
        "import transformers"
      ],
      "metadata": {
        "id": "OzTiUc4vUdDW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "try:\n",
        "    # TPU detection. No parameters necessary if TPU_NAME environment variable is set (always set in Kaggle)\n",
        "    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n",
        "    tf.config.experimental_connect_to_cluster(tpu)\n",
        "    tf.tpu.experimental.initialize_tpu_system(tpu)\n",
        "    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n",
        "    print('Running on TPU ', tpu.master())\n",
        "except ValueError:\n",
        "    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n",
        "    strategy = tf.distribute.get_strategy()\n",
        "\n",
        "print('Number of replicas:', strategy.num_replicas_in_sync)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l9CEMUninBGc",
        "outputId": "80770fa2-8f2a-4db5-91ee-4b86154daf65"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:absl:Entering into master device scope: /job:worker/replica:0/task:0/device:CPU:0\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Deallocate tpu buffers before initializing tpu system.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Deallocate tpu buffers before initializing tpu system.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.3.243.146:8470\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.3.243.146:8470\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n",
            "WARNING:absl:`tf.distribute.experimental.TPUStrategy` is deprecated, please use  the non experimental symbol `tf.distribute.TPUStrategy` instead.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Running on TPU  grpc://10.3.243.146:8470\n",
            "Number of replicas: 8\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "xlnet_model = 'xlnet-large-cased'\n",
        "xlnet_tokenizer = XLNetTokenizer.from_pretrained(xlnet_model)"
      ],
      "metadata": {
        "id": "GqswXTDvY-20",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "ab5a0f39b374453f97d65f2cb6454c07",
            "dc0fe31513954da5aa1708e7230c18ec",
            "62c4854a1305487f998e4ba0d65b9e91",
            "38aa137682af4c78b4c3e140405b65c3",
            "607af8ca61134e4db9aa6aa6828b3d76",
            "f59babf93dc34608843d7a078a3d717c",
            "62d7243a3113495a94605c5217331b34",
            "87c23e85ece748faa765c1a669e06a9d",
            "4be046a6ef4f4b7999cfc926fb93d2b9",
            "40d1000a9fef4045aa34136f967ecc80",
            "eec03b58bac7479fb2677e79ae57d252",
            "da9e888309fd465d8cdbf022af128296",
            "0b952341bbde46fca5970f470d10c6d3",
            "b8f9d9ce77454b5eb96d5b1a8b033b66",
            "b620d458522c47fd8494c52bf46aa8db",
            "c6f992f372974692b20b293fb5a2ca8d",
            "b5e463792f9940df8979550fc645a5c4",
            "113539e35b0d4756a5dc866503ab1601",
            "6b7f508035f1460d8b3398caeeacc8f0",
            "acecf30fb58246f68927c94a31258022",
            "9faae80009cb483db0134cb2f8f7d54a",
            "ed7dd3b49e8e48fc875712c561460d66",
            "85ed8e75a5de4676a57fdc57a19a2f78",
            "16d78c976f564839b1e23b07a10951ac",
            "9370800b70c0453e85603c3dbc2a9209",
            "7394e1276fd54769a0f69501adf64b15",
            "59e002e2678846b087fa240a8d129435",
            "d0606f59320542429a7e39d8e9bf8ba5",
            "199abf1bb30541c8b4ba7b1ab4b618bb",
            "32786184f6d84f37bd1b040113957692",
            "1f2a05bce0f7464d94f3cd5cf459afc1",
            "77a0e785fa7c4f19b6f517c2d13705d1",
            "70c2952ddc2245b48906769adef31fcf"
          ]
        },
        "outputId": "73e18fb2-48f3-49f2-9c7d-1a96d0131060"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ab5a0f39b374453f97d65f2cb6454c07",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/779k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "da9e888309fd465d8cdbf022af128296",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.32M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "85ed8e75a5de4676a57fdc57a19a2f78",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/761 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def create_xlnet(mname):\n",
        "    \"\"\" Creates the model. It is composed of the XLNet main block and then\n",
        "    a classification head its added\n",
        "    \"\"\"\n",
        "    # Define token ids as inputs\n",
        "    word_inputs = tf.keras.Input(shape=(120,), name='word_inputs', dtype='int32')\n",
        "\n",
        "    # Call XLNet model\n",
        "    xlnet = TFXLNetModel.from_pretrained(mname)\n",
        "    xlnet_encodings = xlnet(word_inputs)[0]\n",
        "\n",
        "    # CLASSIFICATION HEAD \n",
        "    # Collect last step from last hidden state (CLS)\n",
        "    doc_encoding = tf.squeeze(xlnet_encodings[:, -1:, :], axis=1)\n",
        "    # Apply dropout for regularization\n",
        "    doc_encoding = tf.keras.layers.Dropout(.1)(doc_encoding)\n",
        "    net = tf.keras.layers.Dense(200, activation='relu')(doc_encoding)\n",
        "    net = tf.keras.layers.Dropout(0.2)(net)\n",
        "    # Final output \n",
        "    outputs = tf.keras.layers.Dense(3, activation='softmax', name='outputs')(net)\n",
        "\n",
        "    # Compile model\n",
        "    model = tf.keras.Model(inputs=[word_inputs], outputs=[outputs])\n",
        "    model.compile(optimizer=tf.keras.optimizers.Adam(lr=2e-5), loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n",
        "\n",
        "    return model\n"
      ],
      "metadata": {
        "id": "ASR00j6BZOm4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "xlnet = create_xlnet(xlnet_model)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 197,
          "referenced_widgets": [
            "da9eca3ff8ea4f01969ede286fc67dab",
            "f68454de03884ae588c3ec5250c14a80",
            "9cb2793b151a4461acc875b8943a5230",
            "1e961f27ec624f6aa37b2d44b50c53b6",
            "8a179d20736e471795d36e8d87e654c6",
            "eb4cdbb857644dc5a1abb32a41e7d9ad",
            "68d4850031c8495abbccb0ab1cb96fb8",
            "fad48accdbda4ef3875c84a0e1e0222d",
            "d045688efd784f67b9a3d7e64e047ba5",
            "f3ff1041d8ea4c8ba3abff13bb6b3e1c",
            "50edf4507d5c455eade09c0a13d31dc2"
          ]
        },
        "id": "tMBTcs55Zmim",
        "outputId": "153659f6-94ee-40c8-ad85-9e00265fddf8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "da9eca3ff8ea4f01969ede286fc67dab",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.46G [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some layers from the model checkpoint at xlnet-large-cased were not used when initializing TFXLNetModel: ['lm_loss']\n",
            "- This IS expected if you are initializing TFXLNetModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFXLNetModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "All the layers of TFXLNetModel were initialized from the model checkpoint at xlnet-large-cased.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFXLNetModel for predictions without further training.\n",
            "/usr/local/lib/python3.7/dist-packages/keras/optimizer_v2/adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  super(Adam, self).__init__(name, **kwargs)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "xlnet.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "26VVq5SnaHT9",
        "outputId": "f9fefa5d-ad2f-4e51-c9c6-a45803006c05"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " word_inputs (InputLayer)    [(None, 120)]             0         \n",
            "                                                                 \n",
            " tfxl_net_model (TFXLNetMode  TFXLNetModelOutput(last_  360268800\n",
            " l)                          hidden_state=(None, 120,            \n",
            "                              1024),                             \n",
            "                              mems=((120, None, 1024)            \n",
            "                             , (120, None, 1024),                \n",
            "                              (120, None, 1024),                 \n",
            "                              (120, None, 1024),                 \n",
            "                              (120, None, 1024),                 \n",
            "                              (120, None, 1024),                 \n",
            "                              (120, None, 1024),                 \n",
            "                              (120, None, 1024),                 \n",
            "                              (120, None, 1024),                 \n",
            "                              (120, None, 1024),                 \n",
            "                              (120, None, 1024),                 \n",
            "                              (120, None, 1024),                 \n",
            "                              (120, None, 1024),                 \n",
            "                              (120, None, 1024),                 \n",
            "                              (120, None, 1024),                 \n",
            "                              (120, None, 1024),                 \n",
            "                              (120, None, 1024),                 \n",
            "                              (120, None, 1024),                 \n",
            "                              (120, None, 1024),                 \n",
            "                              (120, None, 1024),                 \n",
            "                              (120, None, 1024),                 \n",
            "                              (120, None, 1024),                 \n",
            "                              (120, None, 1024),                 \n",
            "                              (120, None, 1024)),                \n",
            "                              hidden_states=None, att            \n",
            "                             entions=None)                       \n",
            "                                                                 \n",
            " tf.__operators__.getitem (S  (None, 1, 1024)          0         \n",
            " licingOpLambda)                                                 \n",
            "                                                                 \n",
            " tf.compat.v1.squeeze (TFOpL  (None, 1024)             0         \n",
            " ambda)                                                          \n",
            "                                                                 \n",
            " dropout_73 (Dropout)        (None, 1024)              0         \n",
            "                                                                 \n",
            " dense (Dense)               (None, 200)               205000    \n",
            "                                                                 \n",
            " dropout_74 (Dropout)        (None, 200)               0         \n",
            "                                                                 \n",
            " outputs (Dense)             (None, 3)                 603       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 360,474,403\n",
            "Trainable params: 360,474,403\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "X_train,X_test, y_train,y_test = train_test_split(df.cleanData,df.search_word, test_size = 0.2, stratify = df.search_word,random_state = 142)"
      ],
      "metadata": {
        "id": "AknKsQBraXZS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_inputs(text, tokenizer, max_len=120):\n",
        "    \"\"\" Gets tensors from text using the tokenizer provided\"\"\"\n",
        "    inps = [tokenizer.encode_plus(t, max_length=max_len, padding='max_length', add_special_tokens=True) for t in text]\n",
        "    inp_tok = np.array([a['input_ids'] for a in inps])\n",
        "    ids = np.array([a['attention_mask'] for a in inps])\n",
        "    segments = np.array([a['token_type_ids'] for a in inps])\n",
        "    return inp_tok, ids, segments"
      ],
      "metadata": {
        "id": "Ru_shMCOafZX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "inp_tok, ids, segments = get_inputs(X_train, xlnet_tokenizer)"
      ],
      "metadata": {
        "id": "TBu1W3JtanZz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
        "\n",
        "\n",
        "es = EarlyStopping(monitor = 'val_loss', mode = 'min', verbose = 1, patience = 8)\n",
        "mc = ModelCheckpoint('./model_xlnet.h5', monitor = 'val_accuracy', mode = 'max', verbose = 1, save_best_only = True)"
      ],
      "metadata": {
        "id": "GWW0zx5Tb8-h"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "out_tok, ids_out, segments_out = get_inputs(X_test, xlnet_tokenizer)"
      ],
      "metadata": {
        "id": "T_vqSDindYQ0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "hist = xlnet.fit(inp_tok,y_train, epochs = 25, batch_size = 200,verbose = 1,validation_split=.2, callbacks= [es, mc])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CPbniRH7cCGH",
        "outputId": "f6da9f61-6a5c-4e5e-98e3-f926e25d361c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/25\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['tfxl_net_model/transformer/mask_emb:0', 'tfxl_net_model/transformer/layer_._0/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._0/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._1/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._1/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._2/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._2/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._3/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._3/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._4/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._4/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._5/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._5/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._6/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._6/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._7/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._7/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._8/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._8/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._9/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._9/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._10/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._10/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._11/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._11/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._12/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._12/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._13/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._13/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._14/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._14/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._15/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._15/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._16/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._16/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._17/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._17/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._18/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._18/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._19/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._19/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._20/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._20/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._21/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._21/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._22/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._22/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._23/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._23/rel_attn/seg_embed:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss`argument?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:tensorflow:Gradients do not exist for variables ['tfxl_net_model/transformer/mask_emb:0', 'tfxl_net_model/transformer/layer_._0/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._0/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._1/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._1/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._2/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._2/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._3/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._3/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._4/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._4/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._5/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._5/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._6/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._6/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._7/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._7/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._8/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._8/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._9/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._9/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._10/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._10/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._11/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._11/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._12/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._12/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._13/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._13/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._14/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._14/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._15/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._15/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._16/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._16/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._17/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._17/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._18/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._18/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._19/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._19/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._20/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._20/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._21/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._21/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._22/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._22/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._23/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._23/rel_attn/seg_embed:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss`argument?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "WARNING:tensorflow:Gradients do not exist for variables ['tfxl_net_model/transformer/mask_emb:0', 'tfxl_net_model/transformer/layer_._0/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._0/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._1/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._1/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._2/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._2/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._3/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._3/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._4/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._4/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._5/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._5/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._6/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._6/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._7/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._7/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._8/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._8/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._9/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._9/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._10/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._10/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._11/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._11/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._12/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._12/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._13/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._13/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._14/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._14/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._15/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._15/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._16/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._16/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._17/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._17/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._18/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._18/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._19/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._19/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._20/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._20/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._21/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._21/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._22/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._22/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._23/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._23/rel_attn/seg_embed:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss`argument?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:tensorflow:Gradients do not exist for variables ['tfxl_net_model/transformer/mask_emb:0', 'tfxl_net_model/transformer/layer_._0/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._0/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._1/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._1/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._2/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._2/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._3/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._3/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._4/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._4/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._5/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._5/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._6/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._6/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._7/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._7/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._8/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._8/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._9/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._9/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._10/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._10/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._11/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._11/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._12/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._12/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._13/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._13/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._14/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._14/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._15/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._15/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._16/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._16/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._17/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._17/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._18/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._18/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._19/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._19/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._20/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._20/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._21/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._21/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._22/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._22/rel_attn/seg_embed:0', 'tfxl_net_model/transformer/layer_._23/rel_attn/r_s_bias:0', 'tfxl_net_model/transformer/layer_._23/rel_attn/seg_embed:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss`argument?\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "8/8 [==============================] - ETA: 0s - loss: 1.6608 - accuracy: 0.7441  \n",
            "Epoch 00001: val_accuracy improved from -inf to 0.85533, saving model to ./model_xlnet.h5\n",
            "8/8 [==============================] - 787s 97s/step - loss: 1.6608 - accuracy: 0.7441 - val_loss: 0.8038 - val_accuracy: 0.8553\n",
            "Epoch 2/25\n",
            "8/8 [==============================] - ETA: 0s - loss: 0.6504 - accuracy: 0.7994  \n",
            "Epoch 00002: val_accuracy did not improve from 0.85533\n",
            "8/8 [==============================] - 702s 89s/step - loss: 0.6504 - accuracy: 0.7994 - val_loss: 0.5057 - val_accuracy: 0.8553\n",
            "Epoch 3/25\n",
            "8/8 [==============================] - ETA: 0s - loss: 0.5909 - accuracy: 0.8216  \n",
            "Epoch 00003: val_accuracy did not improve from 0.85533\n",
            "8/8 [==============================] - 703s 89s/step - loss: 0.5909 - accuracy: 0.8216 - val_loss: 0.4793 - val_accuracy: 0.8553\n",
            "Epoch 4/25\n",
            "8/8 [==============================] - ETA: 0s - loss: 0.5374 - accuracy: 0.8311  \n",
            "Epoch 00004: val_accuracy did not improve from 0.85533\n",
            "8/8 [==============================] - 703s 89s/step - loss: 0.5374 - accuracy: 0.8311 - val_loss: 0.4965 - val_accuracy: 0.8553\n",
            "Epoch 5/25\n",
            "8/8 [==============================] - ETA: 0s - loss: 0.5390 - accuracy: 0.8375  \n",
            "Epoch 00005: val_accuracy did not improve from 0.85533\n",
            "8/8 [==============================] - 703s 89s/step - loss: 0.5390 - accuracy: 0.8375 - val_loss: 0.4591 - val_accuracy: 0.8553\n",
            "Epoch 6/25\n",
            "8/8 [==============================] - ETA: 0s - loss: 0.5333 - accuracy: 0.8362  \n",
            "Epoch 00006: val_accuracy did not improve from 0.85533\n",
            "8/8 [==============================] - 704s 89s/step - loss: 0.5333 - accuracy: 0.8362 - val_loss: 0.4298 - val_accuracy: 0.8553\n",
            "Epoch 7/25\n",
            "8/8 [==============================] - ETA: 0s - loss: 0.4860 - accuracy: 0.8419  \n",
            "Epoch 00007: val_accuracy did not improve from 0.85533\n",
            "8/8 [==============================] - 705s 89s/step - loss: 0.4860 - accuracy: 0.8419 - val_loss: 0.4696 - val_accuracy: 0.8553\n",
            "Epoch 8/25\n",
            "8/8 [==============================] - ETA: 0s - loss: 0.4962 - accuracy: 0.8349  \n",
            "Epoch 00008: val_accuracy did not improve from 0.85533\n",
            "8/8 [==============================] - 704s 89s/step - loss: 0.4962 - accuracy: 0.8349 - val_loss: 0.4119 - val_accuracy: 0.8553\n",
            "Epoch 9/25\n",
            "8/8 [==============================] - ETA: 0s - loss: 0.4782 - accuracy: 0.8343  \n",
            "Epoch 00009: val_accuracy did not improve from 0.85533\n",
            "8/8 [==============================] - 704s 89s/step - loss: 0.4782 - accuracy: 0.8343 - val_loss: 0.3973 - val_accuracy: 0.8553\n",
            "Epoch 10/25\n",
            "8/8 [==============================] - ETA: 0s - loss: 0.4370 - accuracy: 0.8419  \n",
            "Epoch 00010: val_accuracy did not improve from 0.85533\n",
            "8/8 [==============================] - 705s 89s/step - loss: 0.4370 - accuracy: 0.8419 - val_loss: 0.4260 - val_accuracy: 0.8553\n",
            "Epoch 11/25\n",
            "8/8 [==============================] - ETA: 0s - loss: 0.4235 - accuracy: 0.8432  \n",
            "Epoch 00011: val_accuracy did not improve from 0.85533\n",
            "8/8 [==============================] - 705s 89s/step - loss: 0.4235 - accuracy: 0.8432 - val_loss: 0.5053 - val_accuracy: 0.8553\n",
            "Epoch 12/25\n",
            "8/8 [==============================] - ETA: 0s - loss: 0.3694 - accuracy: 0.8603  \n",
            "Epoch 00012: val_accuracy improved from 0.85533 to 0.88071, saving model to ./model_xlnet.h5\n",
            "8/8 [==============================] - 785s 101s/step - loss: 0.3694 - accuracy: 0.8603 - val_loss: 0.3139 - val_accuracy: 0.8807\n",
            "Epoch 13/25\n",
            "8/8 [==============================] - ETA: 0s - loss: 0.2759 - accuracy: 0.9105  \n",
            "Epoch 00013: val_accuracy improved from 0.88071 to 0.92893, saving model to ./model_xlnet.h5\n",
            "8/8 [==============================] - 782s 100s/step - loss: 0.2759 - accuracy: 0.9105 - val_loss: 0.3253 - val_accuracy: 0.9289\n",
            "Epoch 14/25\n",
            "8/8 [==============================] - ETA: 0s - loss: 0.2707 - accuracy: 0.9168  \n",
            "Epoch 00014: val_accuracy improved from 0.92893 to 0.93909, saving model to ./model_xlnet.h5\n",
            "8/8 [==============================] - 769s 98s/step - loss: 0.2707 - accuracy: 0.9168 - val_loss: 0.2383 - val_accuracy: 0.9391\n",
            "Epoch 15/25\n",
            "8/8 [==============================] - ETA: 0s - loss: 0.2342 - accuracy: 0.9346  \n",
            "Epoch 00015: val_accuracy did not improve from 0.93909\n",
            "8/8 [==============================] - 706s 89s/step - loss: 0.2342 - accuracy: 0.9346 - val_loss: 0.1995 - val_accuracy: 0.9340\n",
            "Epoch 16/25\n",
            "8/8 [==============================] - ETA: 0s - loss: 0.1949 - accuracy: 0.9416  \n",
            "Epoch 00016: val_accuracy improved from 0.93909 to 0.94162, saving model to ./model_xlnet.h5\n",
            "8/8 [==============================] - 763s 97s/step - loss: 0.1949 - accuracy: 0.9416 - val_loss: 0.1914 - val_accuracy: 0.9416\n",
            "Epoch 17/25\n",
            "8/8 [==============================] - ETA: 0s - loss: 0.1952 - accuracy: 0.9410  \n",
            "Epoch 00017: val_accuracy did not improve from 0.94162\n",
            "8/8 [==============================] - 707s 89s/step - loss: 0.1952 - accuracy: 0.9410 - val_loss: 0.2104 - val_accuracy: 0.9239\n",
            "Epoch 18/25\n",
            "8/8 [==============================] - ETA: 0s - loss: 0.1894 - accuracy: 0.9321  \n",
            "Epoch 00018: val_accuracy did not improve from 0.94162\n",
            "8/8 [==============================] - 707s 89s/step - loss: 0.1894 - accuracy: 0.9321 - val_loss: 0.2763 - val_accuracy: 0.9391\n",
            "Epoch 19/25\n",
            "8/8 [==============================] - ETA: 0s - loss: 0.1782 - accuracy: 0.9410  \n",
            "Epoch 00019: val_accuracy did not improve from 0.94162\n",
            "8/8 [==============================] - 708s 89s/step - loss: 0.1782 - accuracy: 0.9410 - val_loss: 0.2180 - val_accuracy: 0.9416\n",
            "Epoch 20/25\n",
            "8/8 [==============================] - ETA: 0s - loss: 0.1515 - accuracy: 0.9467  \n",
            "Epoch 00020: val_accuracy improved from 0.94162 to 0.94416, saving model to ./model_xlnet.h5\n",
            "8/8 [==============================] - 765s 98s/step - loss: 0.1515 - accuracy: 0.9467 - val_loss: 0.1966 - val_accuracy: 0.9442\n",
            "Epoch 21/25\n",
            "8/8 [==============================] - ETA: 0s - loss: 0.1420 - accuracy: 0.9473  \n",
            "Epoch 00021: val_accuracy improved from 0.94416 to 0.95431, saving model to ./model_xlnet.h5\n",
            "8/8 [==============================] - 763s 97s/step - loss: 0.1420 - accuracy: 0.9473 - val_loss: 0.2205 - val_accuracy: 0.9543\n",
            "Epoch 22/25\n",
            "8/8 [==============================] - ETA: 0s - loss: 0.1135 - accuracy: 0.9575  \n",
            "Epoch 00022: val_accuracy did not improve from 0.95431\n",
            "8/8 [==============================] - 709s 90s/step - loss: 0.1135 - accuracy: 0.9575 - val_loss: 0.2366 - val_accuracy: 0.9492\n",
            "Epoch 23/25\n",
            "8/8 [==============================] - ETA: 0s - loss: 0.1144 - accuracy: 0.9619  \n",
            "Epoch 00023: val_accuracy improved from 0.95431 to 0.95685, saving model to ./model_xlnet.h5\n",
            "8/8 [==============================] - 765s 98s/step - loss: 0.1144 - accuracy: 0.9619 - val_loss: 0.2367 - val_accuracy: 0.9569\n",
            "Epoch 24/25\n",
            "8/8 [==============================] - ETA: 0s - loss: 0.1081 - accuracy: 0.9581  \n",
            "Epoch 00024: val_accuracy did not improve from 0.95685\n",
            "8/8 [==============================] - 708s 90s/step - loss: 0.1081 - accuracy: 0.9581 - val_loss: 0.1992 - val_accuracy: 0.9492\n",
            "Epoch 00024: early stopping\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import classification_report\n",
        "\n",
        "y_pred=xlnet.predict(out_tok)\n",
        "predicted = np.argmax(y_pred, axis=1)\n",
        "report = classification_report(y_test, predicted)\n",
        "print(report)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0QU2T_qmoqt2",
        "outputId": "869181cc-889b-4d1a-e799-9be9834fa6de"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.91      0.76      0.83        66\n",
            "           1       0.95      0.99      0.97       418\n",
            "           2       1.00      0.44      0.62         9\n",
            "\n",
            "    accuracy                           0.95       493\n",
            "   macro avg       0.95      0.73      0.80       493\n",
            "weighted avg       0.95      0.95      0.95       493\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "plt.plot(hist.history['accuracy'],c='b',label='train accuracy')\n",
        "plt.plot(hist.history['val_accuracy'],c='r',label='validation accuracy')\n",
        "plt.xlabel(\"Epochs\")\n",
        "plt.ylabel(\"Accuracy\")\n",
        "plt.legend(loc='lower right')\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 278
        },
        "id": "b5cw1wOVmK2A",
        "outputId": "74321be7-6180-4ae1-c6bb-e5ff17e468d3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3iUVfbA8e+hhk4MoECQoIu00ENZRUURRVFEyoLiIiwrq1LEVVfWsqCs5WdBRcEVXOyKiCIgsSXC4q6ACaiAIB2lCaETSiDJ+f1xJ2ESJn1Kyvk8zzwz87Y5Mwxz8t773nNFVTHGGGOyKxfqAIwxxhRPliCMMcb4ZAnCGGOMT5YgjDHG+GQJwhhjjE8VQh2Av9SpU0ejoqJCHYYxxpQoK1as2KeqdX2tKzUJIioqisTExFCHYYwxJYqI/JLTOmtiMsYY45MlCGOMMT5ZgjDGGOOTJQhjjDE+WYIwxhjjkyUIY4wxPlmCMMYY41OpGQdhjDEl2YYN8MUXEBEBV14J550X6ogsQRhjTEikpcHSpTB/vrutX591fatW0KOHu11+OdSqFfwYLUEYY0yQJCfDl1+6hLBwIezbBxUrQvfuMHo09O4N+/dDfLy7zZgBU6ZAuXIQE3MmYVxyCYSFBT5eKS0zysXExKiV2jDGFDc7d8KCBS4pfP01pKRA7douGfTpA9dck/PZQUqKO8uIj3f7Ll/uzjwqV3ZJIiNhdOwIFQr5576IrFDVGJ/rLEEYY4x/HTsGL74Ic+dCxs/SBRfAjTe6pHDJJe7MoaCOHoUlS86cYaxa5Za3bw8rVxYu1twShDUxGWOMHy1fDrfeCps2Qdeu8OSTLim0aAEiOeyUng4ffgiTJ8OFF8LEiXDRRWdtVqOGO/Po3ds937sXFi2C06cD817sMldjjPGD06dhwgR3dnDqFCxe7JqHxo+Hli1zSA6q8Omn0KEDDB4Mhw7BvHluhz//GX79NdfXrFcPBg1yCSkQLEEYY0xuli2DBx90nQBpaT43Wb/eJYbHHoMhQ1zTz+WX53HcRYvg4ovhhhtc7/W778LatbBlC4waBW+/DU2bwrhxsGeP/99Xfqhqqbh17NhRjTHGb06dUn34YdVy5VTd3/qq9eurjhununy5anq6pqerTp2qWqWK6jnnqH74YT6Ou3y56lVXueM1bKj66qvutbL75RfVESNUy5dXrVpV9cEHVQ8c8PvbBBI1h9/VgP5oA72A9cAmYLyP9Y2BeGAVsBiI9FqXBvzguc3P67UsQRhj/GbdOtWOHd1P5G23qe7erfrBB6p9+6pWqqQKejrqQn3vgoe0JWv0mmtUd+7M45irVqneeKM7Zp06qpMnq544kXcs69erDh7s9qtdW/WJJ1STk/3xLlU1RAkCKA9sBi4AKgE/Ai2zbfMhcJvn8ZXA217rkgvyepYgjDFFlp6u+tJLqmFhqhERqnPmnL3NwYOacNdM/bpiT03FnV2kt27tfri3bDl7+40bVYcMURVRrVlTddIk1SNHCh7b99+rXn+9+9muV0/1xRdVT54s+HGyCVWC+D3whdfzvwN/z7bNT0Ajz2MBjnitswRhjAmenTtVr77a/Sxee63qrl1nbXLokOrQoW6TmBjVDd/8pjpliurvf6+ZzVC//71b9sMPqiNHuiaiKlVUH3hAdf/+osf57beq3bu71zr/fNXXXlM9fbrQhwtVghgAvOb1/I/Ay9m2eQ+42/O4H6BAhOd5KpAILAP65vAaIz3bJJ5//vmF/oCMMWXc7Nmq4eHuh3zaNHcmkc1//qPauLHrknjkER/dBlu3qj75pGqbNmeSRcWKqqNH+0w2RZKervrVV6qdOrnX6dzZZ8z5kVuCCNhAOREZAPRS1T97nv8R6KKqo722aQC8DDQBlgD9gWhVPSQiDVV1p4hcAHwN9FDVzTm9ng2UM8YU2KFDMGYMvPMOdOpE+ptvc7xRM44dI8vt44/h2WfdYLd33nHjG3K1dq27zvW66yAqKnDxq7rLYg8ehOHDC3WIUA2U2wk08noe6VmWSVV34c4cEJHqQH9VPeRZt9Nzv0VEFgPtcX0axhhTYL/8Aq+/DitWuB/9ZrsX84/NQ6l7ehfPV53ApNUPcbRlzsObR46E556D6tXz8WItW7pboIlA374BO3wgE0QC0FREmuASw2DgFu8NRKQOcEBV03F9FDM9y8OB46qa4tnmEuDpAMZqjCmFTp1ydZBmzHBF8gA6tDzJvQcfZtCuyfxW7Xf8s8f/2BPVhZHVoFo1lwCqVct6i4yE1q1D+15CIWAJQlVTRWQ08AXuiqaZqvqTiDyGa/OaD3QHnhQRxTUxjfLs3gJ4VUTScYP5nlLVtYGK1RhTuqxfD//+N7z5pitHEdkgnefu2sqQ6B+pN20i7FoNd9xBg2efZWK1aqEOt9iyYn3GmFLh+HH4aI7y0bQ9HFu+mjayhusaraZj5TXU2vkTcvy42/Dcc2HmTNc/YKxYnzGm4DQtnWOHTlOldmXKlw91ND4cPQpr1vDrZ2vY8slqyq9dTa+0NfyRfW69AinnQtNo6D0SoqNdO1Hr1lClSkhDLyksQRhjskg9rSy9/2Ma/OsRKqccoTnL2B8WeVa7fPZbRtv9eefBn/4ENWv6KaBTp+Dnn2HNGli9GtasQVevRn75BYDzgXOoxm8R0Zzu2hft2Rpp7UkGdev6KYiyyRKEMQaAlJPK1w98QeSrD3Npygo2V25B3cpH+C6iL9MGLeFgStWzLv/87TfOWnbqFDzzDDz/PAwcmEuJ6+zS02Hr1iyJgNWr3WTNqalumwoVOBHVnCXJF/MfRnK0cWtihkVzw+jG/K6O1R71u5wGSJS0m42kNmXCTz+p9u+vevfdqsuWFXpwlLfkZNUPRi/RpZUuVQXdUamJJt79lqadSlWdP9+ViBg8ON+vtXy5aocObvxWz56ulFCuduxQvfJKV5AuY4AZqDZponrDDa5I3Xvv6ZFvV+s9o1K0XDlXaeLtt/3y9ss8QlWsL5g3SxCmVEtLc7V3wsJUa9VSrVzZ/fe94AL3A7pmTYEPefCg6mt3JmpcxV6qoEmV6uvaMdM0/WRK1g2ffNK91hNP5PvYqamqL7/sSg9VquRGHh8/7mPD335TbdZMtUYN1bFjVWfMUF26NEutovR01ffeUz3vPJerRo1ysRv/sARhTEm2ffuZ8tC9e7vKoocOqb7+uqsdlFGOOreCcV727lV9fuRanVthgCro4Yrn6Ja7nlE9dsz3Dunpqjff7H6d580rUOi7d6veeuuZE4KFC71W7t/vYq5aVfWbb3zuv26dO7nIqH2UkFCglzf5YAnCmJJq1ixX4rlqVTdvgK82ld9+cxVIL75YM5tnunZ1Zxy7d2dutn276qPDtuhb5W/TVMrpsQo1dNfICaqHD+cdx/Hjrvx19eqFOlv5+mvV5s1daDfdpPrr6kPuF79yZVdTKJtjx9yJUcWK7u2/8oo7KzH+ZwnCmJLmwAHVW25x/0W7dFHdsCF/+23bpvrUU6pt27p9y5XT9Kuu0o+vm6FTy43SFCpqSvkw3fen+1STkgoW0/btrp3nggtU9+0r8FtKSXGtVRFhyfq/cpdoarkKenrugrO2W7BANSpKM6di2LOnwC9lCsAShDElSVycamSkKxP92GOFL+W8dq3qI4/owTq/UwU9LRX08JA7XKdwYS1d6joVrrjC9yxoeTlxQo93u0pTKacDmK2tWrkqqaout2XMp+O93ASWJQhjSoITJ1Tvucf9t2zWTPW774p8yC+/VC0n6Xp39x80besvfghSVd9808U4enTB9ktJOTPhzRtv6Pz5Z84Urr3WVdquWlX16acLl3tM4ViCMKa4+/571ZYt3X/JUaNy7jAugA0b3BQH0dGFm8AsV/fd52J99dX8bX/6tOrAgW6fadMyFx87pvrQQ+6k5KabVH/91c9xmjxZgjCmuEpNdX0GFSuq1q+v+tlnfjnsoUOqLVq4WTPzuKipcFJTVXv1Uq1QIe+2oLS0M9OwPfusz01SUnwuNkGQW4KwoYfGhMrhw3DFFTB+PNx4oxs13KtXkQ+blgZDhsDGjTBnDjRp4odYsytfHt5/Hy68EPr3d5Mt+KIKo0bBW2/Bo4/Cvff63KxSpQDEaIrMEoQxofLOO/DNN64u9ezZEBHhl8M+/DAsXAhTpkD37n45pG+1a8P8+XD6NPTpA8nJWderwv33w7/+BX/7GzzySACDMYFgCcKYUJk/Hy66yFW2y3fBoty99x489RTccQfceadfDpm7iy6CDz5wdZOGDXP1lDJMnOimYBs92gXlp/dogscShDGhcPSom7P4hhv8dsiEBBgxAi6/HF580W+Hzds117gJmz/6CCZNcsv+7//gscdc8nvxRUsOJZRVczUmFL780pU97dPHL4fbvdtNTXzeefDhhyFo0x83DlatcmcNmza55rObb4bp06Gc/R1aUlmCMCYU5s+H8HC4+OIiH+rkSbjpJtfn/e23IZoCQcT1Naxf75LDjTe6+T6L5UxDJr8sQRgTbGlprhe5d2+oULT/gqowciQsX+5aeNq08VOMhVG5Msyb505hRoyAihVDGIzxB0sQxgTb0qWwf79f+h8mT4a333ZXkPbr54fYiqpuXbjrrlBHYfzEGgeNCbYFC9yZwzXXFOkwn33mrh4dMMBd2mqMv1mCMCbYFixwAxRq1Sr0IX7+GQYPdtMuv/GG9QObwLCvlTHBtGkTrFtXpOalgwddH3BGk3+1an6MzxgvliCMCaYFC9x9IRPExo2uSWnrVvj4Y2jc2I+xGZONJQhjgmn+fIiOLlCBJFWIi4Prr4dmzVx1junToVu3AMZpDJYgjAmegwfdr3s+zx5OnIAZM1w/Q8+ebqT0P/4Bv/7qqloYE2h2masxwfL5524MRB6jp3fsgGnT3FnC/v3Qrp3riB482PU7GBMsliCMCZb586FePejc2efqZctc2aI5c1zNu7594e674dJLrZSRCQ1LEMYEw+nTbuBCv35Zrkk9fdqNgH7hBTcaumZNGDvWFUANyDwOxhSAJQhjguGbb1yxJK/mpW3b4LLLYPt2aNoUXnoJbrsNatQIXZjGeLMEYUwwLFjgOhB69sxc9M47Ljl88onrt7bBbqa4sQRhTKCpugTRo0eWUW0LF0KnTm7QmzHFkf3NYkygrVsHmzdnubx13z7X53DddSGMy5g8WIIwJtAyRk9ff33moi++cCcWvXuHKCZj8sEShDGBNn8+dOgAkZGZixYudJWxO3YMYVzG5MEShDGBlJTk5n/wal5KS3Nj5q691jqmTfEW0K+niPQSkfUisklExvtY31hE4kVklYgsFpFIr3W3ichGz+22QMZpTMDExrq2JK8EsWyZq7phzUumuAtYghCR8sBU4FqgJXCziLTMttmzwFuq2gZ4DHjSs+85wASgC9AZmCAi4YGK1ZiAmT8fGjRwTUwesbFuquarrw5hXMbkQyDPIDoDm1R1i6qeAmYB2S/oawl87Xm8yGv9NcBXqnpAVQ8CXwG9AhirMf538qTrjb7hhiy1MmJj4ZJLoHbtEMZmTD4EMkE0BLZ7Pd/hWebtRyBjJt2bgBoiEpHPfRGRkSKSKCKJSUlJfgvcGL9YvBiOHcsyenrnTvjhB7u81ZQMoe4iuw+4XES+By4HdgJp+d1ZVaeraoyqxtStWzdQMRpTOAsWQNWqcOWVmYs++8zdW/+DKQkCOZJ6J9DI63mkZ1kmVd2F5wxCRKoD/VX1kIjsBLpn23dxAGM1xr8yRk/37AlhYZmLFy6ERo2gVasQxmZMPgXyDCIBaCoiTUSkEjAYmO+9gYjUEZGMGP4OzPQ8/gK4WkTCPZ3TV3uWGVMy/PijK7Tk1byUkuJmhrvuOivfbUqGgCUIVU0FRuN+2NcBs1X1JxF5TEQy/td0B9aLyAbgXOBxz74HgEm4JJMAPOZZZkzJsGCBywJebUn//S8kJ1vzkik5RFVDHYNfxMTEaGJiYqjDMMbp1AkqVHCD5Dz++lc3U9z+/Vlq9hkTUiKyQlVjfK0LdSe1MaXPrl2QmHjW3NOxsdC9uyUHU3JYgjDG3z791N17JYjNm2H9eru81ZQsliCM8bcFCyAqCqKjMxfFxrp7SxCmJLEEYYw/HT/uLlXyMXr6oovgd78LYWzGFJAlCGP8KS7Oldjwal46dgwWLbKzB1PyWIIwxp8WLIAaNeDyyzMXLVrkxkDY5a2mpLEEYYy/pKe7BNGrF1SqlLl44UJ35dKll4YwNmMKwRKEMf6SmAh79mQZPa3q+h969oTKlUMYmzGFYAnCGH9ZsMBNEXfttZmL1q6FX3+1/gdTMlmCMMZf5s93Ez1ERGQuWrjQ3XvlDGNKDEsQxvjDL7/AqlVZmpfANS+1bQuRkTnsZ0wxZgnCGH+YO9fde13eeuiQK9BnzUumpLIEYUxRHTkCTz0FF18MzZplLv7qK0hLs8tbTckVyAmDjCkbnnjCXb20YEGWxbGxEB4OXbqEKC5jisjOIIwpis2b4fnnYehQV+LbIz3dJYhevVzVb2NKIksQxhTF/fdDxYrw5JNZFq9cCXv3Wv+DKdksQRhTWIsWuc7pv/8dGjTIsmrhQler75prQhSbMX6QZ4IQkRu85o02xoDrfR43Dho3dlPFZRMb6/oe6tYNQWzG+El+fvgHARtF5GkRaR7ogIwpEV57zY17eOYZqFIly6q9eyEhwZqXTMmXZ4JQ1VuB9sBm4A0RWSoiI0WkRsCjM6Y4OnQIHn7YVd8bMOCs1Z9/7mow2eWtpqTLV9ORqh4B5gCzgPrATcBKERkTwNiMKZ4mTYL9++HFF7NMCpQhNhbOOw/atQtBbMb4UX76IPqIyFxgMVAR6Kyq1wJtgXsDG54xxcyGDTBlCvzpT9C+/VmrU1Phiy9c81I567kzJVx+rtDuDzyvqku8F6rqcREZEZiwjCmm7r3X9Tk8/rjP1UuXuhYo638wpUF+EsREYHfGExGpApyrqttUNT5QgRlT7Hz5JXz6Kfzf/8G55/rcJDbWDYy76qogx2ZMAOTnJPhDIN3reZpnmTFlR2oq3HMPXHgh3H13jpstXOj6rmvVCmJsxgRIfhJEBVU9lfHE87hSLtsbU/r8619u9p9nn81xarjt22H1amteMqVHfhJEkohkFrkXkRuBfYELyZhi5sABmDABevSAG2/McbPYWHdvl7ea0iI/fRB3AO+KyMuAANuBoQGNypjiZOJE1/P8/PM+L2vNEBsLUVHQ3IaTmlIizwShqpuBriJS3fM8OeBRGVNcrF0L06bBX/4CrVvnuFlKCsTFwfDhueYQY0qUfBUiFpHeQCsgTDzfflV9LIBxGRN6qq7OUo0a8FjuX/f//AeOH7f+B1O65JkgRORfQFXgCuA1YADwXYDjMib0YmPdqLfnn4c6dXLc7ORJePRRqF4duncPXnjGBFp+OqkvVtWhwEFVfRT4PXBRYMMyJsROnXJnD82awahROW6mCiNGwLffwsyZULVqEGM0JsDy08R00nN/XEQaAPtx9ZiMKb1eftmV1Vi40E0IlINJk+C999ysowMHBjE+Y4IgPwligYjUBp4BVgIKzAhoVMaEUlKS63Po1SvXToX33nNXv952G4wfH8T4jAmSXBOEZ6KgeFU9BHwkIp8CYap6OCjRBcPBg3D99aGOwhQne/dCcjJMnpzjJv/7n7ti6fLLYfp0u3LJlE65JghVTReRqbj5IFDVFCAlGIEFjYg1HJusoqLggQegRQufq7dsgb593WRyH30ElayugCml8tPEFC8i/YGPVVUDHVDQ1a4NX30V6ihMCXHokDvhTEtz3RMREaGOyJjAyc9VTH/BFedLEZEjInJURI7k5+Ai0ktE1ovIJhE5q5VWRM4XkUUi8r2IrBKR6zzLo0TkhIj84Ln9q0DvypgAOH3adURv2gRz50LTpqGOyJjAys9I6kJNLSoi5YGpQE9gB5AgIvNVda3XZg8Ds1X1FRFpCcQCUZ51m1XV5uQyxYIqjB7tRku//rrrezCmtMvPQLnLfC3PPoGQD52BTaq6xXOcWcCNgHeCUKCm53EtYFde8RgTCs8/7zqj//53GDYs1NEYExz56YO43+txGO6HfwVwZR77NcQV9suwA+iSbZuJwJeeua2rAd7TrDQRke+BI8DDqvpN9hcQkZHASIDzzz8/zzdiTGHMmwf33QcDBsA//xnqaIwJnvw0Md3g/VxEGgEv+On1bwbeUNXnROT3wNsiEo2bwe58Vd0vIh2BT0Sklapm6ftQ1enAdICYmJjS14FuQm7lSrjlFujUCd56y+aZNmVLYb7uOwDf1/9ltRNo5PU80rPM2whgNoCqLsWdodRR1RRV3e9ZvgLYjJX3MEG2cyfccIMrwzRvnpuK2piyJD99EC/h+grAJZR2uBHVeUkAmopIE1xiGAzckm2bX4EewBsi0gKXIJJEpC5wQFXTROQCoCmwJR+vaYxfJCe75HD0qBsUd955oY7ImODLTx9EotfjVOB9Vf1fXjupaqqIjAa+AMoDM1X1JxF5DEhU1fnAvcAMEbkHl4SGqap6OsYfE5HTuPmw71DVAwV7a8YUTloaDBkCP/4In36a6zQQxpRqktfYNxGpBpxU1TTP8/JAZVU9HoT48i0mJkYTExPz3tCUeWlpsG+fq6ixdy/s2XPm8d698PPP7qzh5ZdzLeRqTKkgIitUNcbXunyNpMZdXZQxk1wV4EvgYv+EZ0qqgwdh1iw3gGzYMKhZM89dgmrfPpg61f3geyeBffvcuIbsKlSAevXc7fHHLTkYk58EEeY9zaiqJouIFS8qo1RhyRJ47TWYM8dNlgOuqunYse4W6vITR464OnuTJ8OxY3Dhhe5H/6KL4NJLzySBevXg3HPPPK5d265SMsZbfhLEMRHpoKorATyXnZ4IbFimuNmzB9580yWGjRvd2cLw4fDnP7smmyeecBWyn3sO7rgD7r0X6gd51pDjx90Zw1NPwYED0L+/i6lly+DGYUypoaq53oBOuMtMvwH+C2wCOua1X7BvHTt2VONfqamqsbGq/fqpVqigCqrduqm+8YbqsWNnb796teqQIarlyqlWrqx6552qW7cGPs6UFNVp01Tr13cx9uqlmpgY+Nc1pjTAXTTk83c1z05qABGpCDTzPF2vqqcDkq2KwDqpz9iwAd5/302Elr0ZpV49qFYt9/1/+cXVG5o5E7Zvh7p13aQ4I0ZA8+Z5v/7mzfD00/DGG2euCBo/Psfq2YWWlnZm0p6tW6FbN9d3cJnP4jDGGF9y66TOz1VMo4B31U0ahIiEAzer6jS/R1oEZT1BqLqq5S++CLGxbpqLnP5pq1Xz3QYfHu6K0X35pdvu6qtdE1KfPoWb82DnTnj2WXj1VddX0a8fPPggdOhQ+PcJ7n3NnQuPPAJr10L79i4x9OplE/cYU1BFTRA/aLaqqiLyvaq292OMRVZWE8Tx4/D22y4xrFvnfujvusv1A9Ss6WbP9HUpp69laWkQGQl/+pPrX4iK8k+MSUkuvpdfhsOH3Q/56NFu8Fn16i5hZdxyS0QZSfChhyAx0Z3NTJrkEo91LhtTOEVNEKuBNp62qoxxEKtUtZXfIy2CspYgtm93HbLTp7vLTdu3h3HjYNAgqFy54MdLT3eT4dSqBeXL+z9ecMlh2jRXGTUpyfc2FSqcnTQybgcOQEKCm8lt4kS49Va3vTGm8IqaIJ4BGgOvehb9BfhVVe/za5RFVBYShCosXer+Gv/oI/e8b1+XGLp1KznNK8ePw7JlrozFsWNZb8nJOS9LT3fjLW6/vXBJ0BhztqIOlHsAV1L7Ds/zVYBVpgmiU6fgww9dYkhIcH/ljxvnmmn81QwUTFWrwpV5FYs3xoRcfsp9p4vIcuBC4A9AHeCjQAdm3BnCW2+5SWp273YDvaZOhaFDXTOMMcYEUo4JQkQuws3XcDOwD/gAQFWvCE5oZduvv8Jf/gKffw6//z38+99wzTXWGWuMCZ7cziB+xg2Ou15VNwF4qq6aAEpPd5eF/u1v7gxiyhRXE8gSgzEm2HL72emHm9ltkYjMEJEeQAnpBi2ZNm6EK65wl6l27Qpr1sCYMZYcjDGhkeNPj6p+oqqDgebAImAcUE9EXhGRq4MVYFmQluZqGLVp4+YgeO01N1itJHZAG2NKjzz/NlXVY6r6nrq5qSOB73FXNhk/+OknuPhiuO8+N3J57VpX0qKkXLJqjCm9CtR4oaoHVXW6qvYIVEBlxalTrtJo+/awZYurnfTJJ9CgQagjM8YYx8ahhsCKFa6cxapVcPPNbnxD3bqhjsoYY7Ky7s8gOnnSVTXt0sWVmpg3z1UjteRgjCmO7AwiSFRh8GCXFEaMcFVOa9cOdVTGGJMzO4MIkvffd8nhmWfcVUqWHIwxxZ0liCDYs8eNZ+jaFe6xoYbGmBLCEkQQjBnjKpLOnBm4UtrGGONv1gcRYB9/7CqxPv64/6fcNMaYQLIziAA6cMCVzWjfHu6/P9TRGGNMwdgZRADdcw/s3+8qslasGOpojDGmYOwMIkBiY8/M5dCuXd7bG2NMcWMJIgAOH3ZzObRqBQ89FOpojDGmcKyJKQD+9jfYtcvNG21zJxtjSio7g/Cz+HiYPh3uvRc6dw51NMYYU3iWIPwoORluvx2aNoVHHw11NMYYUzTWxORHDz0E27bBkiVQpUqoozHGmKKxMwg/+e9/4aWXYPRo6NYt1NEYY0zRWYLwgxMnXIXWxo3hiSdCHY0xxviHNTH5wcSJsGEDxMVB9eqhjsYYY/zDziCKKCHBze1w++3QwyZiNcaUIgFNECLSS0TWi8gmERnvY/35IrJIRL4XkVUicp3Xur979lsvItcEMs7CSkmB4cOhfn03z4MxxpQmAWtiEpHywFSgJ7ADSBCR+aq61muzh4HZqvqKiLQEYoEoz+PBQCugARAnIhepalqg4i2MJ56An36CTz+FWrVCHY0xxvhXIM8gOgObVHWLqp4CZgE3ZttGgZqex7WAXZ7HNwKzVDVFVbcCmzzHKzZ+/NEliD/+EXr3DnU0xhjjf4FMEA2B7V7Pd3iWeZsI3CoiO3BnD2MKsC8iMlJEEkUkMSkpydRlOroAABK5SURBVF9x58uYMRARAS+8ENSXNcaYoAl1J/XNwBuqGglcB7wtIvmOSVWnq2qMqsbUrVs3YEFmt2+fG/cwahScc07QXtYYY4IqkJe57gQaeT2P9CzzNgLoBaCqS0UkDKiTz31DZtEiUIWePUMdiTHGBE4gzyASgKYi0kREKuE6nedn2+ZXoAeAiLQAwoAkz3aDRaSyiDQBmgLfBTDWAomLg5o1ISYm1JEYY0zgBOwMQlVTRWQ08AVQHpipqj+JyGNAoqrOB+4FZojIPbgO62GqqsBPIjIbWAukAqOK0xVM8fHQvTtUsGGGxphSLKA/caoai+t89l72D6/Ha4FLctj3ceDxQMZXGNu2webNMHZsqCMxxpjACnUndYkTH+/ur7oqtHEYY0ygWYIooLg4N3K6RYtQR2KMMYFlCaIA0tPdGUSPHiAS6miMMSawLEEUwJo1kJRkzUvGmLLBEkQBxMW5e6vaaowpCyxBFEB8PDRrBpGRoY7EGGMCzxJEPp06Bf/5j509GGPKDksQ+fTdd3DsmPU/GGPKDksQ+RQXB+XKuRHUxhhTFliCyKe4OOjYEcLDQx2JMcYEhyWIfDh6FJYvt+YlY0zZYgkiH5YsgdRU66A2xpQtliDyIT4ewsLgEp9lBY0xpnSyBJEPcXEuOYSFhToSY4wJHksQedizB1avtv4HY0zZYwkiD19/7e6t/8EYU9ZYgshDXBzUrg0dOoQ6EmOMCS6bNDMXqi5BXHEFlC8f6miMKZjTp0+zY8cOTp48GepQTDEQFhZGZGQkFStWzPc+liBysXkz/PorPPBAqCMxpuB27NhBjRo1iIqKQmwCkzJNVdm/fz87duygSZMm+d7PmphykTG9qPU/mJLo5MmTREREWHIwiAgREREFPpu0BJGLuDhX2vuii0IdiTGFY8nBZCjMd8ESRA7S090VTFddZdOLGmPKJksQOfjhBzhwwJqXjCmsQ4cOMW3atELte91113Ho0CE/R2QKyhJEDmx6UWOKJrcEkZqamuu+sbGx1K5dOxBhFYmqkp6eHuowgsauYspBfDy0bAn164c6EmOKbtw4d1bsT+3awQsv5Lx+/PjxbN68mXbt2tGzZ0969+7NI488Qnh4OD///DMbNmygb9++bN++nZMnT3L33XczcuRIAKKiokhMTCQ5OZlrr72Wbt268e2339KwYUPmzZtHlSpVsrzWggUL+Oc//8mpU6eIiIjg3Xff5dxzzyU5OZkxY8aQmJiIiDBhwgT69+/P559/zoMPPkhaWhp16tQhPj6eiRMnUr16de677z4AoqOj+fTTTwG45ppr6NKlCytWrCA2NpannnqKhIQETpw4wYABA3j00UcBSEhI4O677+bYsWNUrlyZ+Ph4evfuzZQpU2jXrh0A3bp1Y+rUqbRt29a//yABYAnCh5QU+OYbuP32UEdiTMn11FNPsWbNGn7wZKbFixezcuVK1qxZk3mp5cyZMznnnHM4ceIEnTp1on///kRERGQ5zsaNG3n//feZMWMGf/jDH/joo4+49dZbs2zTrVs3li1bhojw2muv8fTTT/Pcc88xadIkatWqxerVqwE4ePAgSUlJ3H777SxZsoQmTZpw4MCBPN/Lxo0befPNN+natSsAjz/+OOeccw5paWn06NGDVatW0bx5cwYNGsQHH3xAp06dOHLkCFWqVGHEiBG88cYbvPDCC2zYsIGTJ0+WiOQAliB8WroUTpyw5iVTeuT2l34wde7cOct1+FOmTGHu3LkAbN++nY0bN56VIJo0aZL513fHjh3Ztm3bWcfdsWMHgwYNYvfu3Zw6dSrzNeLi4pg1a1bmduHh4SxYsIDLLrssc5tzzjknz7gbN26cmRwAZs+ezfTp00lNTWX37t2sXbsWEaF+/fp06tQJgJo1awIwcOBAJk2axDPPPMPMmTMZNmxYnq9XXFgfhA9xcW7k9OWXhzoSY0qXatWqZT5evHgxcXFxLF26lB9//JH27dv7vE6/cuXKmY/Lly/vs/9izJgxjB49mtWrV/Pqq68WavR4hQoVsvQveB/DO+6tW7fy7LPPEh8fz6pVq+jdu3eur1e1alV69uzJvHnzmD17NkOGDClwbKFiCcKH+Hjo1Alq1Qp1JMaUXDVq1ODo0aM5rj98+DDh4eFUrVqVn3/+mWXLlhX6tQ4fPkzDhg0BePPNNzOX9+zZk6lTp2Y+P3jwIF27dmXJkiVs3boVILOJKSoqipUrVwKwcuXKzPXZHTlyhGrVqlGrVi327NnDZ599BkCzZs3YvXs3CQkJABw9ejQzmf35z39m7NixdOrUifASNG+xJYhsDh+G776z8t7GFFVERASXXHIJ0dHR3H///Wet79WrF6mpqbRo0YLx48dnacIpqIkTJzJw4EA6duxInTp1Mpc//PDDHDx4kOjoaNq2bcuiRYuoW7cu06dPp1+/frRt25ZBgwYB0L9/fw4cOECrVq14+eWXuSiHEbJt27alffv2NG/enFtuuYVLPDOJVapUiQ8++IAxY8bQtm1bevbsmXlm0bFjR2rWrMnw4cML/R5DQVQ11DH4RUxMjCYmJhb5OPPmQd++sHixNTGZkm3dunW0aNEi1GEYYNeuXXTv3p2ff/6ZcuVC93e5r++EiKxQ1Rhf29sZRDbx8VClChThjxljjMn01ltv0aVLFx5//PGQJofCsKuYsomLg8suA69+MWOMKbShQ4cydOjQUIdRKCUrnQXYrl2wbp1d3mqMMWAJIouM8t7WQW2MMZYgsoiPh4gIKCGDHI0xJqACmiBEpJeIrBeRTSIy3sf650XkB89tg4gc8lqX5rVufiDjhDPTi155JZSwfiRjjAmIgP0Uikh5YCpwLdASuFlEWnpvo6r3qGo7VW0HvAR87LX6RMY6Ve0TqDgzrF8PO3da/4MxoVS9enXAXRY6YMAAn9t0796dvC5pf+GFFzh+/HjmcysfXjiB/Fu5M7BJVbeo6ilgFnBjLtvfDLwfwHhyZf0PxhQfDRo0YM6cOYXeP3uCKK7lw3NSXMqKBzJBNAS2ez3f4Vl2FhFpDDQBvvZaHCYiiSKyTET6Bi5MJy4OoqLgggsC/UrGhMC4cdC9u39v48bl+pLjx4/PUuZi4sSJPPvssyQnJ9OjRw86dOhA69atmTdv3ln7btu2jejoaABOnDjB4MGDadGiBTfddBMnTpzI3O7OO+8kJiaGVq1aMWHCBMAVANy1axdXXHEFV1xxBeDKaOzbtw+AyZMnEx0dTXR0NC94qhhu27aNFi1acPvtt9OqVSuuvvrqLK+TYcGCBXTp0oX27dtz1VVXsWfPHgCSk5MZPnw4rVu3pk2bNnz00UcAfP7553To0IG2bdvSw9M8kfE5ZIiOjmbbtm1s27aNZs2aMXToUKKjo9m+fbvP9weurPjFF19M27Zt6dy5M0ePHuWyyy7LrJwLrsLtjz/+mOu/UV6KyziIwcAcVU3zWtZYVXeKyAXA1yKyWlU3e+8kIiOBkQDnn39+oV88LQ0WLYIBA2x6UWP8ZdCgQYwbN45Ro0YBrgLqF198QVhYGHPnzqVmzZrs27ePrl270qdPnxznTH7llVeoWrUq69atY9WqVXTo0CFzna+y22PHjmXy5MksWrQoS9kNgBUrVvD666+zfPlyVJUuXbpw+eWXEx4ebmXFfQhkgtgJNPJ6HulZ5stgYJT3AlXd6bnfIiKLgfbA5mzbTAemgyu1UdhAV6xwNZisecmUWiGo992+fXv27t3Lrl27SEpKIjw8nEaNGnH69GkefPBBlixZQrly5di5cyd79uzhvPPO83mcJUuWMHbsWADatGlDmzZtMtf5KrvtvT67//73v9x0002Z1Vn79evHN998Q58+faysuA+BTBAJQFMRaYJLDIOBW7JvJCLNgXBgqdeycOC4qqaISB3gEuDpQAWa0f9w5ZWBegVjyqaBAwcyZ84cfvvtt8yieO+++y5JSUmsWLGCihUrEhUVVajy3BlltxMSEggPD2fYsGGFOk6G7GXFfTUxjRkzhr/+9a/06dOHxYsXM3HixAK/TkHLiuf3/WUvK75ixYoCx5ZdwPogVDUVGA18AawDZqvqTyLymIh4X5U0GJilWasGtgASReRHYBHwlKquDVSscXHQpg3UqxeoVzCmbBo0aBCzZs1izpw5DBw4EHCluevVq0fFihVZtGgRv/zyS67HuOyyy3jvvfcAWLNmDatWrQJyLrsNOZcav/TSS/nkk084fvw4x44dY+7cuVx66aX5fj9lrax4QPsgVDUWiM227B/Znk/0sd+3QOtAxpbhxAn43//grruC8WrGlC2tWrXi6NGjNGzYkPqeCd6HDBnCDTfcQOvWrYmJiaF58+a5HuPOO+9k+PDhtGjRghYtWtCxY0cga9ntRo0aZZbdBhg5ciS9evWiQYMGLFq0KHN5hw4dGDZsGJ07dwbcD2r79u19Nif5klFWPDw8nCuvvDLzx/3hhx9m1KhRREdHU758eSZMmEC/fv0yy4qnp6dTr149vvrqK/r3789bb71Fq1at6NKlS77Kinu/P++y4idOnKBKlSrExcVRvXp1v5cVL/PlvnfvhvvugxEjrInJlC5W7rvsyausuJX7LqD69eHddy05GGNKtkCUFS8ul7kaY4wpgkCUFS/zZxDGlGalpQnZFF1hvguWIIwppcLCwti/f78lCYOqsn//fsLCwgq0nzUxGVNKRUZGsmPHDpKSkkIdiikGwsLCiIyMLNA+liCMKaUqVqyYOYrXmMKwJiZjjDE+WYIwxhjjkyUIY4wxPpWakdQikgTkXtQld3WAfX4KpySzz8Gxz8Gxz8EpzZ9DY1Wt62tFqUkQRSUiiTkNNy9L7HNw7HNw7HNwyurnYE1MxhhjfLIEYYwxxidLEGdMD3UAxYR9Do59Do59Dk6Z/BysD8IYY4xPdgZhjDHGJ0sQxhhjfCrzCUJEeonIehHZJCLjQx1PqIjINhFZLSI/iEjBp+YrwURkpojsFZE1XsvOEZGvRGSj577oE/wWczl8DhNFZKfne/GDiFwXyhiDQUQaicgiEVkrIj+JyN2e5WXuO1GmE4SIlAemAtcCLYGbRaRlaKMKqStUtV0ZvN77DaBXtmXjgXhVbQrEe56Xdm9w9ucA8Lzne9HOM898aZcK3KuqLYGuwCjP70KZ+06U6QQBdAY2qeoWVT0FzAJuDHFMJshUdQlwINviG4E3PY/fBPoGNagQyOFzKHNUdbeqrvQ8PgqsAxpSBr8TZT1BNAS2ez3f4VlWFinwpYisEJGRoQ6mGDhXVXd7Hv8GnBvKYEJstIis8jRBlfpmFW8iEgW0B5ZTBr8TZT1BmDO6qWoHXHPbKBG5LNQBFRfqrgUvq9eDvwJcCLQDdgPPhTac4BGR6sBHwDhVPeK9rqx8J8p6gtgJNPJ6HulZVuao6k7P/V5gLq75rSzbIyL1ATz3e0McT0io6h5VTVPVdGAGZeR7ISIVccnhXVX92LO4zH0nynqCSACaikgTEakEDAbmhzimoBORaiJSI+MxcDWwJve9Sr35wG2ex7cB80IYS8hk/CB63EQZ+F6IiAD/Btap6mSvVWXuO1HmR1J7Ltt7ASgPzFTVx0McUtCJyAW4swZw09C+V5Y+BxF5H+iOK+m8B5gAfALMBs7HlZH/g6qW6g7cHD6H7rjmJQW2AX/xaocvlUSkG/ANsBpI9yx+ENcPUba+E2U9QRhjjPGtrDcxGWOMyYElCGOMMT5ZgjDGGOOTJQhjjDE+WYIwxhjjkyUIY/IgImle1Ux/8GfVXxGJ8q6eakxxUiHUARhTApxQ1XahDsKYYLMzCGMKyTOHxtOeeTS+E5HfeZZHicjXngJ38SJyvmf5uSIyV0R+9Nwu9hyqvIjM8Mw98KWIVPFsP9YzJ8EqEZkVordpyjBLEMbkrUq2JqZBXusOq2pr4GXciHyAl4A3VbUN8C4wxbN8CvAfVW0LdAB+8ixvCkxV1VbAIaC/Z/l4oL3nOHcE6s0ZkxMbSW1MHkQkWVWr+1i+DbhSVbd4irv9pqoRIrIPqK+qpz3Ld6tqHRFJAiJVNcXrGFHAV55JaBCRB4CKqvpPEfkcSMaV/fhEVZMD/FaNycLOIIwpGs3hcUGkeD1O40zfYG/cjIcdgAQRsT5DE1SWIIwpmkFe90s9j7/FVQYGGIIr/AZumso7wU13KyK1cjqoiJQDGqnqIuABoBZw1lmMMYFkf5EYk7cqIvKD1/PPVTXjUtdwEVmFOwu42bNsDPC6iNwPJAHDPcvvBqaLyAjcmcKduEl4fCkPvONJIgJMUdVDfntHxuSD9UEYU0iePogYVd0X6liMCQRrYjLGGOOTnUEYY4zxyc4gjDHG+GQJwhhjjE+WIIwxxvhkCcIYY4xPliCMMcb49P9d5jxTBi/rJQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    }
  ]
}