02_1_DistilBERT.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "02.1-DistilBERT.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "authorship_tag": "ABX9TyMFSNGrouesTiFeTI9XNikV",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "8849c5196f4948aa911774ec571e333e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4f96e320aef844c69b00b4d58b0473d0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_51f628cdd3664d30914a8120635d01d5",
              "IPY_MODEL_0884c2477ad54a2e86cba862ad683eb9"
            ]
          }
        },
        "4f96e320aef844c69b00b4d58b0473d0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "51f628cdd3664d30914a8120635d01d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d6f0d6418506401bb48fdc9da42c4f9b",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ad72e860461a42348740422cbd847b4a"
          }
        },
        "0884c2477ad54a2e86cba862ad683eb9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f9c97c8b6d984cb4b6c841914d9e0cff",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 1.68MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dc370df6ba144416a6bfc920a470baa0"
          }
        },
        "d6f0d6418506401bb48fdc9da42c4f9b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ad72e860461a42348740422cbd847b4a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f9c97c8b6d984cb4b6c841914d9e0cff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dc370df6ba144416a6bfc920a470baa0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b1bc537777b04072bd675108bfe100d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6a579a06e4bb4eb4a95547dcb5b03294",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7c31bb637f244fc1aeef4c8b3f735f10",
              "IPY_MODEL_5bb8505df6b5419b8b3ba655e9f5a5ab"
            ]
          }
        },
        "6a579a06e4bb4eb4a95547dcb5b03294": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7c31bb637f244fc1aeef4c8b3f735f10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0e094a6d8bae4e5382383ae05ca32620",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 442,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 442,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4f3db1474a6344ffba078f6709f69af4"
          }
        },
        "5bb8505df6b5419b8b3ba655e9f5a5ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5cd631aafeaf4f38890c682cb8cd0138",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 442/442 [00:04&lt;00:00, 98.5B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7c3170a8bc1449859445b505da8a79a6"
          }
        },
        "0e094a6d8bae4e5382383ae05ca32620": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4f3db1474a6344ffba078f6709f69af4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5cd631aafeaf4f38890c682cb8cd0138": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7c3170a8bc1449859445b505da8a79a6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6de3ecc71c6d4d9bb6ebdbe5df25e60b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_703f8adf4c45475e8175202fdcb323e9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a5fc12a7363d4bd2b46c88fb654e83c0",
              "IPY_MODEL_e1f09119647f4378b777ef34ebe1284b"
            ]
          }
        },
        "703f8adf4c45475e8175202fdcb323e9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a5fc12a7363d4bd2b46c88fb654e83c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_dcd3fccae0aa499d801fa3d0d867bbdd",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 267967963,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 267967963,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e88a7944ad814bd8ae53e7dfff16af16"
          }
        },
        "e1f09119647f4378b777ef34ebe1284b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3bab43f026f1463cb995afebde91dc1c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 268M/268M [00:04&lt;00:00, 65.7MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5b5c31df59704899aab9f172869105e2"
          }
        },
        "dcd3fccae0aa499d801fa3d0d867bbdd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e88a7944ad814bd8ae53e7dfff16af16": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3bab43f026f1463cb995afebde91dc1c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5b5c31df59704899aab9f172869105e2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/valerielimyh/Intent_Recognition_using_BERT/blob/master/02_1_DistilBERT.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dbcImJb2p-Mq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import torch\n",
        "import transformers as ppb # pytorch transformers\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn import metrics\n",
        "from sklearn.model_selection import cross_val_score\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.model_selection import train_test_split, RandomizedSearchCV,KFold\n",
        "seed = 123\n",
        "\n",
        "#settings\n",
        "pd.set_option('display.max_colwidth', -1)\n",
        "np.set_printoptions(threshold=np.inf)\n",
        "pd.options.display.max_columns = None\n",
        "pd.options.display.max_rows = None\n",
        "from IPython.core.interactiveshell import InteractiveShell  \n",
        "InteractiveShell.ast_node_interactivity = \"all\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cq14yQwHp-7t",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 230
        },
        "outputId": "d65208d1-8eef-410c-8fd9-dda49eb1d115"
      },
      "source": [
        "!gdown --id 1OlcvGWReJMuyYQuOZm149vHWwPtlboR6 --output train.csv\n",
        "!gdown --id 1Oi5cRlTybuIF2Fl5Bfsr-KkqrXrdt77w --output valid.csv\n",
        "!gdown --id 1ep9H6-HvhB4utJRLVcLzieWNUSG3P_uF --output test.csv"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1OlcvGWReJMuyYQuOZm149vHWwPtlboR6\n",
            "To: /content/train.csv\n",
            "100% 799k/799k [00:00<00:00, 50.1MB/s]\n",
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1Oi5cRlTybuIF2Fl5Bfsr-KkqrXrdt77w\n",
            "To: /content/valid.csv\n",
            "100% 43.3k/43.3k [00:00<00:00, 65.3MB/s]\n",
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1ep9H6-HvhB4utJRLVcLzieWNUSG3P_uF\n",
            "To: /content/test.csv\n",
            "100% 43.1k/43.1k [00:00<00:00, 67.7MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lHy1__YSqsjK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 166,
          "referenced_widgets": [
            "8849c5196f4948aa911774ec571e333e",
            "4f96e320aef844c69b00b4d58b0473d0",
            "51f628cdd3664d30914a8120635d01d5",
            "0884c2477ad54a2e86cba862ad683eb9",
            "d6f0d6418506401bb48fdc9da42c4f9b",
            "ad72e860461a42348740422cbd847b4a",
            "f9c97c8b6d984cb4b6c841914d9e0cff",
            "dc370df6ba144416a6bfc920a470baa0",
            "b1bc537777b04072bd675108bfe100d4",
            "6a579a06e4bb4eb4a95547dcb5b03294",
            "7c31bb637f244fc1aeef4c8b3f735f10",
            "5bb8505df6b5419b8b3ba655e9f5a5ab",
            "0e094a6d8bae4e5382383ae05ca32620",
            "4f3db1474a6344ffba078f6709f69af4",
            "5cd631aafeaf4f38890c682cb8cd0138",
            "7c3170a8bc1449859445b505da8a79a6",
            "6de3ecc71c6d4d9bb6ebdbe5df25e60b",
            "703f8adf4c45475e8175202fdcb323e9",
            "a5fc12a7363d4bd2b46c88fb654e83c0",
            "e1f09119647f4378b777ef34ebe1284b",
            "dcd3fccae0aa499d801fa3d0d867bbdd",
            "e88a7944ad814bd8ae53e7dfff16af16",
            "3bab43f026f1463cb995afebde91dc1c",
            "5b5c31df59704899aab9f172869105e2"
          ]
        },
        "outputId": "7ce2ec11-2d01-42a7-d712-ec32261506f4"
      },
      "source": [
        "model_class, tokenizer_class, pretrained_weights = (ppb.DistilBertModel, ppb.DistilBertTokenizer, 'distilbert-base-uncased')\n",
        "\n",
        "# Load pretrained model/tokenizer\n",
        "tokenizer = tokenizer_class.from_pretrained(pretrained_weights)\n",
        "model = model_class.from_pretrained(pretrained_weights)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8849c5196f4948aa911774ec571e333e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=231508, style=ProgressStyle(description_wid…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b1bc537777b04072bd675108bfe100d4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=442, style=ProgressStyle(description_width=…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6de3ecc71c6d4d9bb6ebdbe5df25e60b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=267967963, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HBrXxEdbq54z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train = pd.read_csv(\"train.csv\")\n",
        "valid = pd.read_csv(\"valid.csv\")\n",
        "test = pd.read_csv(\"test.csv\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j23asN3Vqzhc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "ec1c277f-026d-4fda-bf04-e8151a4834e8"
      },
      "source": [
        "df = pd.concat([train,valid,test])\n",
        "df.shape"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(14484, 2)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AOI2JpwF9Rqs",
        "colab_type": "text"
      },
      "source": [
        "# Let's preprocess our data so that it matches the data BERT was trained on. \n",
        "\n",
        "For this, we'll \n",
        "\n",
        "1. Lowercase our text (if we're using a BERT lowercase model)\n",
        "2. Tokenize it (i.e. \"sally says hi\" -> [\"sally\", \"says\", \"hi\"])\n",
        "3. Break words into WordPieces (i.e. \"calling\" -> [\"call\", \"##ing\"])\n",
        "4. Map our words to indexes using a vocab file that BERT provides\n",
        "5. Add special \"CLS\" and \"SEP\" tokens "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UulexqQPq0_V",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokenized = df['text'].apply((lambda x: tokenizer.encode(x, add_special_tokens=True)))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pFZcd1EH9Yze",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2JuzfMDW9gC-",
        "colab_type": "text"
      },
      "source": [
        "## Padding \n",
        "After tokenization, tokenized is a list of sentences -- each sentences is represented as a list of tokens. We want BERT to process our examples all at once (as one batch). It's just faster that way. For that reason, we need to pad all lists to the same size, so we can represent the input as one 2-d array, rather than a list of lists (of different lengths)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lGQC26tqq2uU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "34f3b340-a194-4aa4-eb93-d3681694f2a4"
      },
      "source": [
        "max_len = 0\n",
        "for i in tokenized.values:\n",
        "    if len(i) > max_len:\n",
        "        max_len = len(i)\n",
        "\n",
        "padded = np.array([i + [0]*(max_len-len(i)) for i in tokenized.values])\n",
        "np.array(padded).shape"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(14484, 38)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zsUQw-ba9r99",
        "colab_type": "text"
      },
      "source": [
        "## Masking\n",
        "If we directly send `padded` to BERT, that would slightly confuse it. We need to create another variable to tell it to ignore (mask) the padding we've added when it's processing its input. That's what attention_mask is:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i0mkJwynq3hO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "a7215e1a-eddd-4b88-d6fc-453d3c93577a"
      },
      "source": [
        "attention_mask = np.where(padded != 0, 1, 0)\n",
        "attention_mask.shape"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(14484, 38)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ip8tJN3y9zKn",
        "colab_type": "text"
      },
      "source": [
        "The model() function runs our sentences through BERT. The results of the processing will be returned into last_hidden_states."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-IKuUlNSq9s7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#create an input tensor out of the padded token matrix, and send that to DistilBERT\n",
        "input_ids = torch.tensor(np.array(padded))\n",
        "\n",
        "\n",
        "with torch.no_grad():\n",
        "  # last_hidden_states holds the outputs of DistilBERT. \n",
        "  # It is a tuple with the shape (number of examples, max number of tokens in the sequence, number of hidden units in the DistilBERT model). \n",
        "    last_hidden_states = model(input_ids)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DltXmqKVq-jx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        " # Slice the output for the first position for all the sequences, take all hidden unit outputs\n",
        "features = last_hidden_states[0][:,0,:].numpy()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_j_yUR8drFMj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Split our datset into a training set and testing set\n",
        "X_train, X_test, y_train, y_test = train_test_split(features, df['intent'], test_size=0.2, random_state=42)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v7NKlbRUrHUK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "907acbdf-5ad5-4b47-901d-eedc81aecc53"
      },
      "source": [
        "# search for the best value of the C parameter, which determines regularization strength.\n",
        "parameters = {'C': np.linspace(0.0001, 100, 20)}\n",
        "\n",
        "lr_finder = RandomizedSearchCV(estimator = LogisticRegression() , scoring = 'accuracy', \n",
        "                               param_distributions = parameters,\n",
        "                               cv = KFold(n_splits=5, shuffle=True, random_state = seed), \n",
        "                               verbose=50, random_state=seed, n_jobs = -1)                         \n",
        "lr_finder.fit(X_train, y_train)\n",
        "\n",
        "print('best parameters: ', lr_finder.best_params_)\n",
        "print('best scores: ', lr_finder.best_score_)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Fitting 5 folds for each of 10 candidates, totalling 50 fits\n",
            "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
            "[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:   10.1s\n",
            "[Parallel(n_jobs=-1)]: Done   2 tasks      | elapsed:   10.1s\n",
            "[Parallel(n_jobs=-1)]: Done   3 tasks      | elapsed:   10.2s\n",
            "[Parallel(n_jobs=-1)]: Done   4 tasks      | elapsed:   10.6s\n",
            "[Parallel(n_jobs=-1)]: Done   5 tasks      | elapsed:   17.0s\n",
            "[Parallel(n_jobs=-1)]: Done   6 tasks      | elapsed:   17.3s\n",
            "[Parallel(n_jobs=-1)]: Done   7 tasks      | elapsed:   17.3s\n",
            "[Parallel(n_jobs=-1)]: Done   8 tasks      | elapsed:   17.9s\n",
            "[Parallel(n_jobs=-1)]: Done   9 tasks      | elapsed:   24.0s\n",
            "[Parallel(n_jobs=-1)]: Done  10 tasks      | elapsed:   24.1s\n",
            "[Parallel(n_jobs=-1)]: Done  11 tasks      | elapsed:   24.4s\n",
            "[Parallel(n_jobs=-1)]: Done  12 tasks      | elapsed:   25.0s\n",
            "[Parallel(n_jobs=-1)]: Done  13 tasks      | elapsed:   31.3s\n",
            "[Parallel(n_jobs=-1)]: Done  14 tasks      | elapsed:   31.3s\n",
            "[Parallel(n_jobs=-1)]: Done  15 tasks      | elapsed:   31.3s\n",
            "[Parallel(n_jobs=-1)]: Done  16 tasks      | elapsed:   32.1s\n",
            "[Parallel(n_jobs=-1)]: Done  17 tasks      | elapsed:   38.2s\n",
            "[Parallel(n_jobs=-1)]: Done  18 tasks      | elapsed:   38.3s\n",
            "[Parallel(n_jobs=-1)]: Done  19 tasks      | elapsed:   38.5s\n",
            "[Parallel(n_jobs=-1)]: Done  20 tasks      | elapsed:   39.2s\n",
            "[Parallel(n_jobs=-1)]: Done  21 tasks      | elapsed:   45.3s\n",
            "[Parallel(n_jobs=-1)]: Done  22 tasks      | elapsed:   45.3s\n",
            "[Parallel(n_jobs=-1)]: Done  23 tasks      | elapsed:   45.9s\n",
            "[Parallel(n_jobs=-1)]: Done  24 tasks      | elapsed:   46.4s\n",
            "[Parallel(n_jobs=-1)]: Done  25 tasks      | elapsed:   52.3s\n",
            "[Parallel(n_jobs=-1)]: Done  26 tasks      | elapsed:   52.4s\n",
            "[Parallel(n_jobs=-1)]: Done  27 tasks      | elapsed:   52.9s\n",
            "[Parallel(n_jobs=-1)]: Done  28 tasks      | elapsed:   53.9s\n",
            "[Parallel(n_jobs=-1)]: Done  29 tasks      | elapsed:   59.2s\n",
            "[Parallel(n_jobs=-1)]: Done  30 tasks      | elapsed:   59.4s\n",
            "[Parallel(n_jobs=-1)]: Done  31 tasks      | elapsed:   60.0s\n",
            "[Parallel(n_jobs=-1)]: Done  32 tasks      | elapsed:  1.0min\n",
            "[Parallel(n_jobs=-1)]: Done  33 tasks      | elapsed:  1.1min\n",
            "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:  1.1min\n",
            "[Parallel(n_jobs=-1)]: Done  35 tasks      | elapsed:  1.1min\n",
            "[Parallel(n_jobs=-1)]: Done  36 tasks      | elapsed:  1.1min\n",
            "[Parallel(n_jobs=-1)]: Done  37 tasks      | elapsed:  1.2min\n",
            "[Parallel(n_jobs=-1)]: Done  38 tasks      | elapsed:  1.2min\n",
            "[Parallel(n_jobs=-1)]: Done  39 tasks      | elapsed:  1.2min\n",
            "[Parallel(n_jobs=-1)]: Done  40 tasks      | elapsed:  1.3min\n",
            "[Parallel(n_jobs=-1)]: Done  41 tasks      | elapsed:  1.3min\n",
            "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:  1.3min\n",
            "[Parallel(n_jobs=-1)]: Done  43 tasks      | elapsed:  1.3min\n",
            "[Parallel(n_jobs=-1)]: Done  45 out of  50 | elapsed:  1.3min remaining:    8.6s\n",
            "[Parallel(n_jobs=-1)]: Done  47 out of  50 | elapsed:  1.4min remaining:    5.3s\n",
            "[Parallel(n_jobs=-1)]: Done  50 out of  50 | elapsed:  1.5min finished\n",
            "best parameters:  {'C': 21.05271052631579}\n",
            "best scores:  0.9698800887613517\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zt2xFv3qtaJ4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "outputId": "cf6dbf0e-0a3c-4d8c-cc21-fb50f3117a31"
      },
      "source": [
        "lr_cv = lr_finder.best_estimator_.fit(X_train, y_train)\n",
        "y_lr_pred = lr_cv.predict(X_test)\n",
        "print(metrics.classification_report(y_test, y_lr_pred))"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "                      precision    recall  f1-score   support\n",
            "\n",
            "       AddToPlaylist       0.98      0.99      0.98       392\n",
            "      BookRestaurant       0.99      0.99      0.99       379\n",
            "          GetWeather       0.99      1.00      1.00       435\n",
            "           PlayMusic       0.95      0.97      0.96       441\n",
            "            RateBook       0.99      0.99      0.99       399\n",
            "  SearchCreativeWork       0.92      0.91      0.91       409\n",
            "SearchScreeningEvent       0.96      0.94      0.95       442\n",
            "\n",
            "            accuracy                           0.97      2897\n",
            "           macro avg       0.97      0.97      0.97      2897\n",
            "        weighted avg       0.97      0.97      0.97      2897\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G1nAM-cVtwiU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Using DistilBERT to embed text and modeling using Log reg gives us higher accuracy than \n",
        "# using count vectoriser to generate a document-term matrix and modeling using Log reg. "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zyYzdDGXA9l1",
        "colab_type": "text"
      },
      "source": [
        "### References: \n",
        "http://jalammar.github.io/a-visual-guide-to-using-bert-for-the-first-time/\n",
        "\n",
        "https://github.com/snipsco/nlu-benchmark/tree/master/2017-06-custom-intent-engines"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5kGmroNGA8IF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}