goemotions-distilbert.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "bertgoemotions.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "cf91233115474e5da8f6855e42c0802d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ad43bea4e7a7437a8fb444978bde1504",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a81e9333a31a43619eeea0b23fa1f173",
              "IPY_MODEL_8c0a726408964788b1b5520e1e829366"
            ]
          }
        },
        "ad43bea4e7a7437a8fb444978bde1504": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a81e9333a31a43619eeea0b23fa1f173": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ba60f5284ac64127b4e37823dc7870fa",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9120c5aa8ef1490796766ecac6cf60fc"
          }
        },
        "8c0a726408964788b1b5520e1e829366": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1c7c35a1913a4e0a9a6c08b4d7630566",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:20&lt;00:00, 11.2kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_173cff612ef54ac7840334f6d346cfd8"
          }
        },
        "ba60f5284ac64127b4e37823dc7870fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9120c5aa8ef1490796766ecac6cf60fc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1c7c35a1913a4e0a9a6c08b4d7630566": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "173cff612ef54ac7840334f6d346cfd8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f9d2f87ca00044f484b050e5010881a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1cbd04261d984c06bb27632f6956c849",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_934916f46f984abaa77961f203c1d40d",
              "IPY_MODEL_bd4e4e6122514fc3a159acea46fc9768"
            ]
          }
        },
        "1cbd04261d984c06bb27632f6956c849": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "934916f46f984abaa77961f203c1d40d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7d15bdbfe12d43e0bd69b6d7cb537e85",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 466062,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 466062,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_99d8b93081cc424fb49c25a08fb6c54d"
          }
        },
        "bd4e4e6122514fc3a159acea46fc9768": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0d4d2a8a10da44a48992997f31805acd",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 466k/466k [00:01&lt;00:00, 399kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b7b48da903bc47e6a5698bfc275dd66e"
          }
        },
        "7d15bdbfe12d43e0bd69b6d7cb537e85": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "99d8b93081cc424fb49c25a08fb6c54d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0d4d2a8a10da44a48992997f31805acd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b7b48da903bc47e6a5698bfc275dd66e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f831bf9d780f4e68be9ef6a34ec499b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_01160d25405b497f87df1e0fa0fc45d1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5deeb0bde38841419f2117ee399cc055",
              "IPY_MODEL_e1fcb5fe2328491da814e5467ac80877"
            ]
          }
        },
        "01160d25405b497f87df1e0fa0fc45d1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5deeb0bde38841419f2117ee399cc055": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_469bad90427a4a3a8c78784cbec35935",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 28,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 28,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5743955abe0747838a0df2b1a9a03ce9"
          }
        },
        "e1fcb5fe2328491da814e5467ac80877": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3bd4aa50064041dc90db2ef3f05c1ae2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 28.0/28.0 [00:18&lt;00:00, 1.48B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1444d4d56f484980b5e4417ee8363533"
          }
        },
        "469bad90427a4a3a8c78784cbec35935": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5743955abe0747838a0df2b1a9a03ce9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3bd4aa50064041dc90db2ef3f05c1ae2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1444d4d56f484980b5e4417ee8363533": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GY2DuIgGcZzC"
      },
      "source": [
        "#DistilBert on GoEmotions dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tl6tbIsHcf46"
      },
      "source": [
        "#### Download Dataset and label texts\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7F_trMU89QY-",
        "outputId": "a7799437-c4d1-4d5f-e0ec-35da6a9bd95e"
      },
      "source": [
        "!wget https://raw.githubusercontent.com/google-research/google-research/master/goemotions/data/train.tsv\n",
        "!wget https://raw.githubusercontent.com/google-research/google-research/master/goemotions/data/dev.tsv\n",
        "!wget https://raw.githubusercontent.com/google-research/google-research/master/goemotions/data/test.tsv\n",
        "!wget https://raw.githubusercontent.com/google-research/google-research/master/goemotions/data/emotions.txt"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-04-22 02:25:58--  https://raw.githubusercontent.com/google-research/google-research/master/goemotions/data/train.tsv\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 3519053 (3.4M) [text/plain]\n",
            "Saving to: ‘train.tsv’\n",
            "\n",
            "train.tsv           100%[===================>]   3.36M  13.5MB/s    in 0.2s    \n",
            "\n",
            "2021-04-22 02:25:58 (13.5 MB/s) - ‘train.tsv’ saved [3519053/3519053]\n",
            "\n",
            "--2021-04-22 02:25:59--  https://raw.githubusercontent.com/google-research/google-research/master/goemotions/data/dev.tsv\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 439059 (429K) [text/plain]\n",
            "Saving to: ‘dev.tsv’\n",
            "\n",
            "dev.tsv             100%[===================>] 428.77K  --.-KB/s    in 0.05s   \n",
            "\n",
            "2021-04-22 02:25:59 (8.39 MB/s) - ‘dev.tsv’ saved [439059/439059]\n",
            "\n",
            "--2021-04-22 02:25:59--  https://raw.githubusercontent.com/google-research/google-research/master/goemotions/data/test.tsv\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 436706 (426K) [text/plain]\n",
            "Saving to: ‘test.tsv’\n",
            "\n",
            "test.tsv            100%[===================>] 426.47K  --.-KB/s    in 0.04s   \n",
            "\n",
            "2021-04-22 02:25:59 (9.50 MB/s) - ‘test.tsv’ saved [436706/436706]\n",
            "\n",
            "--2021-04-22 02:25:59--  https://raw.githubusercontent.com/google-research/google-research/master/goemotions/data/emotions.txt\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.111.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.111.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 248 [text/plain]\n",
            "Saving to: ‘emotions.txt’\n",
            "\n",
            "emotions.txt        100%[===================>]     248  --.-KB/s    in 0s      \n",
            "\n",
            "2021-04-22 02:25:59 (11.8 MB/s) - ‘emotions.txt’ saved [248/248]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_5giiPaXcx3s"
      },
      "source": [
        "#### Load dataset from tsv"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8OK7Lg-z-9OH"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kP_jslou_3Ip"
      },
      "source": [
        "df_train = pd.read_csv(\"/content/train.tsv\", sep='\\t', header =None)\n",
        "df_test = pd.read_csv(\"/content/test.tsv\", sep='\\t', header =None)\n",
        "df_dev = pd.read_csv(\"/content/dev.tsv\", sep='\\t', header =None)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "l6MbJsNRDO-a",
        "outputId": "344361f5-9370-4ed8-8c67-84d33e23bea3"
      },
      "source": [
        "df_dev.head()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Is this in New Orleans?? I really feel like th...</td>\n",
              "      <td>27</td>\n",
              "      <td>edgurhb</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>You know the answer man, you are programmed to...</td>\n",
              "      <td>4,27</td>\n",
              "      <td>ee84bjg</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>I've never been this sad in my life!</td>\n",
              "      <td>25</td>\n",
              "      <td>edcu99z</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>The economy is heavily controlled and subsidiz...</td>\n",
              "      <td>4,27</td>\n",
              "      <td>edc32e2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>He could have easily taken a real camera from ...</td>\n",
              "      <td>20</td>\n",
              "      <td>eepig6r</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                   0     1        2\n",
              "0  Is this in New Orleans?? I really feel like th...    27  edgurhb\n",
              "1  You know the answer man, you are programmed to...  4,27  ee84bjg\n",
              "2               I've never been this sad in my life!    25  edcu99z\n",
              "3  The economy is heavily controlled and subsidiz...  4,27  edc32e2\n",
              "4  He could have easily taken a real camera from ...    20  eepig6r"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UBG-4B4sc2fh"
      },
      "source": [
        "#### Data Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fx13cRVRAaZu"
      },
      "source": [
        "df_train = df_train.drop([2], axis=1)\n",
        "df_test = df_test.drop([2], axis=1)\n",
        "df_dev = df_dev.drop([2], axis=1)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LAccn1vxBJ5Q"
      },
      "source": [
        "def makeclass(x):\n",
        "  x = x.split(',')\n",
        "  return x[0]"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kwFuE7tMAbWL"
      },
      "source": [
        "df_train[1] = df_train[1].map(makeclass)\n",
        "df_test[1] = df_test[1].map(makeclass)\n",
        "df_dev[1] = df_dev[1].map(makeclass)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B_ApedYaOcxy"
      },
      "source": [
        "df_train.columns = ['text', 'labels']\n",
        "df_test.columns = ['text', 'labels']\n",
        "df_dev.columns = ['text', 'labels']"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "F9q1Rg3aAcrp",
        "outputId": "b3e6aad6-f368-41be-bf1b-d89e8d0786fd"
      },
      "source": [
        "df_test"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>I’m really sorry about your situation :( Altho...</td>\n",
              "      <td>25</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>It's wonderful because it's awful. At not with.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Kings fan here, good luck to you guys! Will be...</td>\n",
              "      <td>13</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>I didn't know that, thank you for teaching me ...</td>\n",
              "      <td>15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>They got bored from haunting earth for thousan...</td>\n",
              "      <td>27</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5422</th>\n",
              "      <td>Thanks. I was diagnosed with BP 1 after the ho...</td>\n",
              "      <td>15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5423</th>\n",
              "      <td>Well that makes sense.</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5424</th>\n",
              "      <td>Daddy issues [NAME]</td>\n",
              "      <td>27</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5425</th>\n",
              "      <td>So glad I discovered that subreddit a couple m...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5426</th>\n",
              "      <td>Had to watch \"Elmo in Grouchland\" one time too...</td>\n",
              "      <td>27</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5427 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                   text labels\n",
              "0     I’m really sorry about your situation :( Altho...     25\n",
              "1       It's wonderful because it's awful. At not with.      0\n",
              "2     Kings fan here, good luck to you guys! Will be...     13\n",
              "3     I didn't know that, thank you for teaching me ...     15\n",
              "4     They got bored from haunting earth for thousan...     27\n",
              "...                                                 ...    ...\n",
              "5422  Thanks. I was diagnosed with BP 1 after the ho...     15\n",
              "5423                             Well that makes sense.      4\n",
              "5424                                Daddy issues [NAME]     27\n",
              "5425  So glad I discovered that subreddit a couple m...      0\n",
              "5426  Had to watch \"Elmo in Grouchland\" one time too...     27\n",
              "\n",
              "[5427 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SSZH-LyeZpTM"
      },
      "source": [
        "# Maps ids to emotion label\n",
        "\n",
        "id2labels = {}\n",
        "with open('/content/emotions.txt', 'r') as f:\n",
        "  lines = f.readlines()\n",
        "  lines = list(map(lambda x: x.strip(\"\\n\"), lines))\n",
        "\n",
        "for i in range(28):\n",
        "  id2labels[str(i)] = lines[i]"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p-02ID-ObJ86",
        "outputId": "bbc6059c-38c3-4a3b-a1c0-be065fc3dc5e"
      },
      "source": [
        "id2labels"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'0': 'admiration',\n",
              " '1': 'amusement',\n",
              " '10': 'disapproval',\n",
              " '11': 'disgust',\n",
              " '12': 'embarrassment',\n",
              " '13': 'excitement',\n",
              " '14': 'fear',\n",
              " '15': 'gratitude',\n",
              " '16': 'grief',\n",
              " '17': 'joy',\n",
              " '18': 'love',\n",
              " '19': 'nervousness',\n",
              " '2': 'anger',\n",
              " '20': 'optimism',\n",
              " '21': 'pride',\n",
              " '22': 'realization',\n",
              " '23': 'relief',\n",
              " '24': 'remorse',\n",
              " '25': 'sadness',\n",
              " '26': 'surprise',\n",
              " '27': 'neutral',\n",
              " '3': 'annoyance',\n",
              " '4': 'approval',\n",
              " '5': 'caring',\n",
              " '6': 'confusion',\n",
              " '7': 'curiosity',\n",
              " '8': 'desire',\n",
              " '9': 'disappointment'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xFoXlFsvLB5l"
      },
      "source": [
        "from sklearn.preprocessing import LabelBinarizer"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9661lPbDLN1L"
      },
      "source": [
        "onehot = LabelBinarizer()"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x3DXPCTcMLO_"
      },
      "source": [
        "train_onehot = onehot.fit_transform(df_train['labels'])\n",
        "test_onehot = onehot.transform(df_test['labels'])\n",
        "val_onehot = onehot.transform(df_dev['labels'])"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wA5O7RhZNJpa"
      },
      "source": [
        "df_train = pd.concat([df_train, pd.DataFrame(train_onehot)], axis=1)\n",
        "df_test = pd.concat([df_test, pd.DataFrame(test_onehot)], axis=1)\n",
        "df_dev = pd.concat([df_dev, pd.DataFrame(val_onehot)], axis=1)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 776
        },
        "id": "x-f367IvNuqI",
        "outputId": "2c0222ae-80f2-4c4f-aca8-2889c6f441ee"
      },
      "source": [
        "df_train"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>labels</th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "      <th>27</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>My favourite food is anything I didn't have to...</td>\n",
              "      <td>27</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Now if he does off himself, everyone will thin...</td>\n",
              "      <td>27</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>WHY THE FUCK IS BAYLESS ISOING</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>To make her feel threatened</td>\n",
              "      <td>14</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Dirty Southern Wankers</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43405</th>\n",
              "      <td>Added you mate well I’ve just got the bow and ...</td>\n",
              "      <td>18</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43406</th>\n",
              "      <td>Always thought that was funny but is it a refe...</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43407</th>\n",
              "      <td>What are you talking about? Anything bad that ...</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43408</th>\n",
              "      <td>More like a baptism, with sexy results!</td>\n",
              "      <td>13</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43409</th>\n",
              "      <td>Enjoy the ride!</td>\n",
              "      <td>17</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>43410 rows × 30 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text labels  0  ...  25  26  27\n",
              "0      My favourite food is anything I didn't have to...     27  0  ...   0   0   0\n",
              "1      Now if he does off himself, everyone will thin...     27  0  ...   0   0   0\n",
              "2                         WHY THE FUCK IS BAYLESS ISOING      2  0  ...   0   0   0\n",
              "3                            To make her feel threatened     14  0  ...   0   0   0\n",
              "4                                 Dirty Southern Wankers      3  0  ...   0   0   0\n",
              "...                                                  ...    ... ..  ...  ..  ..  ..\n",
              "43405  Added you mate well I’ve just got the bow and ...     18  0  ...   0   0   0\n",
              "43406  Always thought that was funny but is it a refe...      6  0  ...   0   0   0\n",
              "43407  What are you talking about? Anything bad that ...      3  0  ...   0   0   0\n",
              "43408            More like a baptism, with sexy results!     13  0  ...   0   0   0\n",
              "43409                                    Enjoy the ride!     17  0  ...   0   0   0\n",
              "\n",
              "[43410 rows x 30 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8SUq-HI-AugZ"
      },
      "source": [
        "# Save processed dataset for future purpose\n",
        "df_train.to_csv(\"/content/train.csv\")\n",
        "df_test.to_csv(\"/content/test.csv\")\n",
        "df_dev.to_csv(\"/content/dev.csv\")"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uHxodc3_dThf"
      },
      "source": [
        "#### Installing Dependencies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hO4eAM0eEVXY",
        "outputId": "48cb9d64-72c9-4c35-c415-0e7074c56c43"
      },
      "source": [
        "! pip install transformers"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d8/b2/57495b5309f09fa501866e225c84532d1fd89536ea62406b2181933fb418/transformers-4.5.1-py3-none-any.whl (2.1MB)\n",
            "\u001b[K     |████████████████████████████████| 2.1MB 8.6MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n",
            "\u001b[K     |████████████████████████████████| 901kB 52.8MB/s \n",
            "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ae/04/5b870f26a858552025a62f1649c20d29d2672c02ff3c3fb4c688ca46467a/tokenizers-0.10.2-cp37-cp37m-manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 56.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (3.10.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Installing collected packages: sacremoses, tokenizers, transformers\n",
            "Successfully installed sacremoses-0.0.45 tokenizers-0.10.2 transformers-4.5.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b2UeZcwSGs5s",
        "outputId": "0e10a50d-9de7-4b25-b1ed-1f5bf5ce59e7"
      },
      "source": [
        "! pip install datasets"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting datasets\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/da/d6/a3d2c55b940a7c556e88f5598b401990805fc0f0a28b2fc9870cf0b8c761/datasets-1.6.0-py3-none-any.whl (202kB)\n",
            "\u001b[K     |████████████████████████████████| 204kB 8.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.1.5)\n",
            "Collecting xxhash\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/4f/0a862cad26aa2ed7a7cd87178cbbfa824fc1383e472d63596a0d018374e7/xxhash-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl (243kB)\n",
            "\u001b[K     |████████████████████████████████| 245kB 34.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.11.1)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2.23.0)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.3)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from datasets) (3.10.1)\n",
            "Requirement already satisfied: tqdm<4.50.0,>=4.27 in /usr/local/lib/python3.7/dist-packages (from datasets) (4.41.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from datasets) (20.9)\n",
            "Collecting huggingface-hub<0.1.0\n",
            "  Downloading https://files.pythonhosted.org/packages/a1/88/7b1e45720ecf59c6c6737ff332f41c955963090a18e72acbcbeac6b25e86/huggingface_hub-0.0.8-py3-none-any.whl\n",
            "Requirement already satisfied: pyarrow>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (3.0.0)\n",
            "Collecting fsspec\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e9/91/2ef649137816850fa4f4c97c6f2eabb1a79bf0aa2c8ed198e387e373455e/fsspec-2021.4.0-py3-none-any.whl (108kB)\n",
            "\u001b[K     |████████████████████████████████| 112kB 37.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets) (1.19.5)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2020.12.5)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->datasets) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->datasets) (3.4.1)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->datasets) (2.4.7)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<0.1.0->datasets) (3.0.12)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Installing collected packages: xxhash, huggingface-hub, fsspec, datasets\n",
            "Successfully installed datasets-1.6.0 fsspec-2021.4.0 huggingface-hub-0.0.8 xxhash-2.0.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Bo_niBcddXr5"
      },
      "source": [
        "#### Model loading"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ADxyRZ4fIoL2"
      },
      "source": [
        "import transformers\n",
        "import datasets"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 164,
          "referenced_widgets": [
            "cf91233115474e5da8f6855e42c0802d",
            "ad43bea4e7a7437a8fb444978bde1504",
            "a81e9333a31a43619eeea0b23fa1f173",
            "8c0a726408964788b1b5520e1e829366",
            "ba60f5284ac64127b4e37823dc7870fa",
            "9120c5aa8ef1490796766ecac6cf60fc",
            "1c7c35a1913a4e0a9a6c08b4d7630566",
            "173cff612ef54ac7840334f6d346cfd8",
            "f9d2f87ca00044f484b050e5010881a6",
            "1cbd04261d984c06bb27632f6956c849",
            "934916f46f984abaa77961f203c1d40d",
            "bd4e4e6122514fc3a159acea46fc9768",
            "7d15bdbfe12d43e0bd69b6d7cb537e85",
            "99d8b93081cc424fb49c25a08fb6c54d",
            "0d4d2a8a10da44a48992997f31805acd",
            "b7b48da903bc47e6a5698bfc275dd66e",
            "f831bf9d780f4e68be9ef6a34ec499b8",
            "01160d25405b497f87df1e0fa0fc45d1",
            "5deeb0bde38841419f2117ee399cc055",
            "e1fcb5fe2328491da814e5467ac80877",
            "469bad90427a4a3a8c78784cbec35935",
            "5743955abe0747838a0df2b1a9a03ce9",
            "3bd4aa50064041dc90db2ef3f05c1ae2",
            "1444d4d56f484980b5e4417ee8363533"
          ]
        },
        "id": "e0n2B8lNAjYH",
        "outputId": "3f52a221-fade-420e-8c41-34f1333b483b"
      },
      "source": [
        "from transformers import DistilBertTokenizerFast\n",
        "tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cf91233115474e5da8f6855e42c0802d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f9d2f87ca00044f484b050e5010881a6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=466062.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f831bf9d780f4e68be9ef6a34ec499b8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=28.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rJ1v_qCXdbj0"
      },
      "source": [
        "#### Data preprocessing for model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QXPO1mXVZ_xM"
      },
      "source": [
        "train_encodings = tokenizer(df_train['text'].tolist(), truncation=True, padding=True)\n",
        "val_encodings = tokenizer(df_dev['text'].tolist(), truncation=True, padding=True)\n",
        "test_encodings = tokenizer(df_test['text'].tolist(), truncation=True, padding=True)"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zS4x_kWsZKzB"
      },
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "train_dataset = tf.data.Dataset.from_tensor_slices((\n",
        "    dict(train_encodings),\n",
        "    df_train.iloc[:,2:]\n",
        "))\n",
        "val_dataset = tf.data.Dataset.from_tensor_slices((\n",
        "    dict(val_encodings),\n",
        "    df_dev.iloc[:,2:]\n",
        "))\n",
        "test_dataset = tf.data.Dataset.from_tensor_slices((\n",
        "    dict(test_encodings),\n",
        "    df_test.iloc[:,2:]\n",
        "))"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tOcQeSoJ4_rG",
        "outputId": "2313aa82-6a0a-4b7b-d885-d54ff381dbbd"
      },
      "source": [
        "train_dataset"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<TensorSliceDataset shapes: ({input_ids: (316,), attention_mask: (316,)}, (28,)), types: ({input_ids: tf.int32, attention_mask: tf.int32}, tf.int64)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Bc_VYXSydfLk"
      },
      "source": [
        "#### Model Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uoE1nbNrAZAb"
      },
      "source": [
        "import tensorflow as tf\n",
        "from transformers import TFDistilBertForSequenceClassification\n",
        "\n",
        "model = TFDistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased', num_labels=28)\n",
        "\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate=5e-5)\n",
        "loss = tf.keras.losses.CategoricalCrossentropy(from_logits=True)\n",
        "model.compile(optimizer=optimizer, loss=loss, metrics=[\"accuracy\"]) # can also use any keras loss fn\n",
        "model.fit(train_dataset.shuffle(1000).batch(16), epochs=3, batch_size=16)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uSVMvrxsVBWK"
      },
      "source": [
        "model.save_pretrained('./pretrained')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6PhzIL1KX3kX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d17b8121-6845-4032-975a-2d33b3d1648a"
      },
      "source": [
        "tokenizer.save_pretrained(\"./pretrained\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('./pretrained/tokenizer_config.json',\n",
              " './pretrained/special_tokens_map.json',\n",
              " './pretrained/vocab.txt',\n",
              " './pretrained/added_tokens.json')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jYTCq2VEgIas"
      },
      "source": [
        "import pickle\n",
        "with open('./pretrained/labelbin.pickle', 'wb') as f:\n",
        "    pickle.dump(onehot, f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l25BQlZD61hg"
      },
      "source": [
        "###Inference"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xgyoDvDVF1Kf",
        "outputId": "fa565686-3b1b-433c-ccc3-c3eccc60c924"
      },
      "source": [
        "model1 = TFDistilBertForSequenceClassification.from_pretrained('./pretrained/')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some layers from the model checkpoint at ./pretrained/ were not used when initializing TFDistilBertForSequenceClassification: ['dropout_19']\n",
            "- This IS expected if you are initializing TFDistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFDistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some layers of TFDistilBertForSequenceClassification were not initialized from the model checkpoint at ./pretrained/ and are newly initialized: ['dropout_39']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ONBITsWu2jGm",
        "outputId": "c3f548d1-56f6-4f72-f78f-08ea0c91f9bc"
      },
      "source": [
        "print(df_test.loc[0,'text'])\n",
        "print(df_test.loc[0,'labels'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "I’m really sorry about your situation :( Although I love the names Sapphira, Cirilla, and Scarlett!\n",
            "25\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FRfD6ZPYIAde"
      },
      "source": [
        "inputs = tokenizer(\"I’m really sorry about your situation :( Although I love the names Sapphira, Cirilla, and Scarlett!\", return_tensors=\"tf\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xYtGo3x2Vdx_"
      },
      "source": [
        "output = model1(inputs)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y2j54OvEVmq7"
      },
      "source": [
        "output = tf.nn.softmax(output['logits'])\n",
        "output.numpy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YDq6nvxgVzC8",
        "outputId": "07e33f6b-ef1c-4354-bef2-f4f6f086705c"
      },
      "source": [
        "out = np.zeros(28, dtype='int')\n",
        "ind = np.argmax(output.numpy())\n",
        "out[ind] = 1\n",
        "finallabel = onehot.inverse_transform(out.reshape([1,28]))\n",
        "print(\"Predicted emotion: \" + id2labels[finallabel[0]])"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Predicted emotion: sadness\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VYP60byhYRL-"
      },
      "source": [
        "### Download Trained Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jKlLhfAVWgZT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6e4fb499-a3c7-4514-8f11-4e51407e6e5a"
      },
      "source": [
        "! zip -r /content/pretrained.zip /content/pretrained"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  adding: content/pretrained/ (stored 0%)\n",
            "  adding: content/pretrained/tokenizer_config.json (deflated 38%)\n",
            "  adding: content/pretrained/tf_model.h5 (deflated 8%)\n",
            "  adding: content/pretrained/labelbin.pickle (deflated 41%)\n",
            "  adding: content/pretrained/config.json (deflated 65%)\n",
            "  adding: content/pretrained/special_tokens_map.json (deflated 40%)\n",
            "  adding: content/pretrained/vocab.txt (deflated 53%)\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}