Bert_kerasTF_multiclassification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Bert_kerasTF_multiclassification.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPIkCgQiQ/XegviVoFaPvrt"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "8d266dcf07b941dd83dc7082d67d6371": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ff2cffa4da4244f781c31033b313c72f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d6f6a47f91194d28b8394d94914b0d8c",
              "IPY_MODEL_cb401d95e8c649cbb8ee508c67eb3ec6"
            ]
          }
        },
        "ff2cffa4da4244f781c31033b313c72f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d6f6a47f91194d28b8394d94914b0d8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cb2420efd61a41d4b3768e7b9e72f9c9",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a8fb9b954558411b993ef1288901df98"
          }
        },
        "cb401d95e8c649cbb8ee508c67eb3ec6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2c34f9bdc5d84d918a8a4e01f8f14b2e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [01:52&lt;00:00, 2.06kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_680c94c2ef3b4753964713d3e10c2e61"
          }
        },
        "cb2420efd61a41d4b3768e7b9e72f9c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a8fb9b954558411b993ef1288901df98": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2c34f9bdc5d84d918a8a4e01f8f14b2e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "680c94c2ef3b4753964713d3e10c2e61": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KHtoyklX7MPE"
      },
      "source": [
        "**Multi-Class classification with BERT using keras and tensorflow**\r\n",
        "\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9lwvLZ0P7VWM",
        "outputId": "de063f3d-583e-43f3-a268-71eeafcc34f7"
      },
      "source": [
        "from google.colab import drive\r\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "irg82YkSEA8H"
      },
      "source": [
        "Load the Text data "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0emfrQ7U7x3k",
        "outputId": "680d047e-f870-4f28-dc1f-8fd29ddaab11"
      },
      "source": [
        "import pandas as pd\r\n",
        "import numpy as np\r\n",
        "from sklearn.model_selection import train_test_split\r\n",
        "\r\n",
        "#The first line in input file contains \"5485\" adding \"skiprows\" skips 1st line\r\n",
        "df = pd.read_csv('/content/gdrive/My Drive/doc_classification.txt', header=None, skiprows=1)\r\n",
        "print(df.head(3))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "                                                   0\n",
            "0  1 champion products ch approves stock split ch...\n",
            "1  2 computer terminal systems cpml completes sal...\n",
            "2  1 cobanco inc cbco year net shr cts vs dlrs ne...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n-I1KQidEJg9"
      },
      "source": [
        "Extract labels and Sentences and create a DataFrame"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aLXhLu1Q8nDJ",
        "outputId": "11d50f7c-b1d9-4b84-93c5-75f3b88eac76"
      },
      "source": [
        "sentences=[]\r\n",
        "labels=[]\r\n",
        "#Iterate through all rows 0th column and extract lables and sentences\r\n",
        "#If maxsplit is specified, the list will have the maximum of maxsplit+1 items, here each line is split into 2 items at 0->label at 1->whole sentence.\r\n",
        "for line in df.iloc[:,0]:\r\n",
        "    line=line.split(\" \", maxsplit=1)\r\n",
        "    labels.append(line[0])\r\n",
        "    sentences.append(line[1])\r\n",
        "\r\n",
        "sentences=pd.DataFrame(sentences)\r\n",
        "labels=pd.DataFrame(labels)\r\n",
        "df2=pd.concat([sentences,labels],axis=1) # Here column names will be '0', so giving names to columns \r\n",
        "df2.columns=[\"Sentence\",\"Label\"]\r\n",
        "\r\n",
        "#Drop duplicates except for first one\r\n",
        "df3=df2.drop_duplicates(keep='first')\r\n",
        "print(\"Before: Class compositions percentage: \\n\",df3[\"Label\"].value_counts(normalize=True))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Before: Class compositions percentage: \n",
            " 1    0.522388\n",
            "2    0.293164\n",
            "6    0.045145\n",
            "3    0.044039\n",
            "8    0.035931\n",
            "7    0.032430\n",
            "4    0.019348\n",
            "5    0.007555\n",
            "Name: Label, dtype: float64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hrppBni6_TYH"
      },
      "source": [
        "Data Pre-processing, cleaning"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 296
        },
        "id": "vEusb0I8_Uye",
        "outputId": "7ea4e58e-b332-48fe-f72b-e3e7c791fb58"
      },
      "source": [
        "import re\r\n",
        "\r\n",
        "# Data Pre-processing. Converting to lower case, Remove special characters.\r\n",
        "def clean_corpus(sentence):    \r\n",
        "     \r\n",
        "        #Covert to lower case        \r\n",
        "        sentence = sentence.lower()  \r\n",
        "        #Remove special characters      \r\n",
        "        pattern1 = r'[\\,+\\:\\?\\!\\\"\\(\\)!\\'\\.\\%\\[\\]]+'\r\n",
        "        # Remove words with 1 and 2 char length\r\n",
        "        pattern2 = r'\\b\\w{1,2}\\b'\r\n",
        "        #Remove extra spaces\r\n",
        "        pattern3 = r' +'   \r\n",
        "\r\n",
        "        replacements=[(pattern1 , \" \"), (pattern2 , \" \"), (pattern3 , \" \")]\r\n",
        "        for pat,repl in replacements:\r\n",
        "            sentence = re.sub(pat, repl, sentence)  \r\n",
        "        return sentence         \r\n",
        "       \r\n",
        "    \r\n",
        "df3[\"Sentence\"] = df3.Sentence.apply(lambda s: clean_corpus(s))\r\n",
        "\r\n",
        "print(\"We have %d lines/Sentences in the corpus.\" %len(df3.Sentence))\r\n",
        "df3.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "We have 5427 lines/Sentences in the corpus.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:18: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sentence</th>\n",
              "      <th>Label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>champion products approves stock split champio...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>computer terminal systems cpml completes sale ...</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>cobanco inc cbco year net shr cts dlrs net ass...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>international inc qtr jan oper shr loss two c...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>brown forman inc bfd qtr net shr one dlr cts n...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                            Sentence Label\n",
              "0  champion products approves stock split champio...     1\n",
              "1  computer terminal systems cpml completes sale ...     2\n",
              "2  cobanco inc cbco year net shr cts dlrs net ass...     1\n",
              "3   international inc qtr jan oper shr loss two c...     1\n",
              "4  brown forman inc bfd qtr net shr one dlr cts n...     1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EtqOZ3oxEvEH"
      },
      "source": [
        "Bert expects labels/categories to start from 0. \r\n",
        "\r\n",
        "Also, \"lablel\" column should be of type int or float. If \"lablel\" column is of type obj OR string, it has to be converted to int or float.\r\n",
        "\r\n",
        "If above two are not handled Bert will not work as expected OR can raise  error(s)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r7TKWEXv-A7A",
        "outputId": "bdf50cb9-8688-4785-dd7b-dcbd8a4b0aaf"
      },
      "source": [
        "df3[\"sentenceLength\"] = df3.Sentence.apply(lambda x : len(x))\r\n",
        "#Each sentence must be classified into one of 8 categories.\r\n",
        "print(df3.Label.unique())\r\n",
        "#BERT expects class labels to start from 0, instead of 1, else Bert will not work.\r\n",
        "df3['label_encode'] = df3['Label'].map({'1':0,'2':1,'3':2,'4':3,'5':4,'6':5,'7':6,'8':7})\r\n",
        "print(df3.label_encode.unique())\r\n",
        "# Half of sentences are of length 337\r\n",
        "print(df3.sentenceLength.describe())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['1' '2' '3' '4' '5' '6' '7' '8']\n",
            "[0 1 2 3 4 5 6 7]\n",
            "count    5427.000000\n",
            "mean      546.466188\n",
            "std       646.976398\n",
            "min        23.000000\n",
            "25%       147.000000\n",
            "50%       337.000000\n",
            "75%       619.000000\n",
            "max      4772.000000\n",
            "Name: sentenceLength, dtype: float64\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  \"\"\"\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "id": "mstQHV3-Tudc",
        "outputId": "00ca1abe-0cb7-4887-b1d4-2b83670f95ce"
      },
      "source": [
        "df_bert = df3.copy()\r\n",
        "df_bert.drop(['sentenceLength'], axis =1, index=None, inplace=True)\r\n",
        "#Labels column were onj type, so converting to int, else Bert will not work. obj/str must be converted to int or float.\r\n",
        "df_bert['label_encode'] = df_bert.label_encode.astype(int)\r\n",
        "#df_bert.reset_index(drop=True) #remove index col\r\n",
        "df_bert.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sentence</th>\n",
              "      <th>Label</th>\n",
              "      <th>label_encode</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>champion products approves stock split champio...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>computer terminal systems cpml completes sale ...</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>cobanco inc cbco year net shr cts dlrs net ass...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>international inc qtr jan oper shr loss two c...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>brown forman inc bfd qtr net shr one dlr cts n...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                            Sentence Label  label_encode\n",
              "0  champion products approves stock split champio...     1             0\n",
              "1  computer terminal systems cpml completes sale ...     2             1\n",
              "2  cobanco inc cbco year net shr cts dlrs net ass...     1             0\n",
              "3   international inc qtr jan oper shr loss two c...     1             0\n",
              "4  brown forman inc bfd qtr net shr one dlr cts n...     1             0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2Dgw7SrmTGeb",
        "outputId": "40ad00b6-2554-4e7e-8066-c015822816d3"
      },
      "source": [
        "import tensorflow as tf\r\n",
        "\r\n",
        "# Get the GPU device name.\r\n",
        "device_name = tf.test.gpu_device_name()\r\n",
        "\r\n",
        "# The device name should look like the following:\r\n",
        "if device_name == '/device:GPU:0':\r\n",
        "    print('Found GPU at: {}'.format(device_name))\r\n",
        "else:\r\n",
        "    raise SystemError('GPU device not found')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found GPU at: /device:GPU:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 657,
          "referenced_widgets": [
            "8d266dcf07b941dd83dc7082d67d6371",
            "ff2cffa4da4244f781c31033b313c72f",
            "d6f6a47f91194d28b8394d94914b0d8c",
            "cb401d95e8c649cbb8ee508c67eb3ec6",
            "cb2420efd61a41d4b3768e7b9e72f9c9",
            "a8fb9b954558411b993ef1288901df98",
            "2c34f9bdc5d84d918a8a4e01f8f14b2e",
            "680c94c2ef3b4753964713d3e10c2e61"
          ]
        },
        "id": "29fIf1sdWD_D",
        "outputId": "04182c91-92e3-40b2-c15c-b143c04d80b3"
      },
      "source": [
        "!pip install transformers\r\n",
        "from transformers import BertTokenizer\r\n",
        "\r\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True, num_labels=8)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/98/87/ef312eef26f5cecd8b17ae9654cdd8d1fae1eb6dbd87257d6d73c128a4d0/transformers-4.3.2-py3-none-any.whl (1.8MB)\n",
            "\u001b[K     |████████████████████████████████| 1.8MB 16.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from transformers) (3.4.0)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 55.5MB/s \n",
            "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fd/5b/44baae602e0a30bcc53fbdbc60bd940c15e143d252d658dfdefce736ece5/tokenizers-0.10.1-cp36-cp36m-manylinux2010_x86_64.whl (3.2MB)\n",
            "\u001b[K     |████████████████████████████████| 3.2MB 50.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.0.0)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893261 sha256=e48880f56c75c56afc7f27dcf94e81a10fa5f614928789aaf2dbc1c193bce3c9\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses, tokenizers, transformers\n",
            "Successfully installed sacremoses-0.0.43 tokenizers-0.10.1 transformers-4.3.2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8d266dcf07b941dd83dc7082d67d6371",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Me4xfymuNUw5"
      },
      "source": [
        "Because the labels are imbalanced, we split the data set in a stratified fashion into Train and valtest set. We will further split valtest set into validation and Test set."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rb5CS54oWUmo",
        "outputId": "85c20be2-cc6c-4115-90f2-207257185a4f"
      },
      "source": [
        "# can be up to 512 for BERT\r\n",
        "max_length = 200\r\n",
        "batch_size = 6\r\n",
        "\r\n",
        "X_train, X_valtest, y_train, y_valtest = train_test_split(df_bert.Sentence.values, \r\n",
        "                                                  df_bert.label_encode.values, \r\n",
        "                                                  test_size=0.2, \r\n",
        "                                                  random_state=42, \r\n",
        "                                                  stratify=df3.Label.values)\r\n",
        "\r\n",
        "print('Type of X_train: ', type(X_train))\r\n",
        "print(X_train[:3])\r\n",
        "print(X_valtest[:3])\r\n",
        "print(y_train[:3])\r\n",
        "print(y_valtest[:3])\r\n",
        "\r\n",
        "print('length of X_train: ', len(X_train))\r\n",
        "print('length of X_test: ', len(X_valtest))\r\n",
        "print('length of y_train: ', len(y_train))\r\n",
        "print('length of y_test: ', len(y_valtest))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Type of X_train:  <class 'numpy.ndarray'>\n",
            "['analysis and technology inc aati hikes payout annual div cts cts prior pay april record march reuter '\n",
            " 'yankee cos ynk unit sell asssets yankee cos inc eskey inc esk subsidiary said reached agreement principle sell its eskey yale key inc subsidiary new concern formed key management and private investor for about mln dlrs part sale eskey said the buyers will assume the mln dlrs publicly held eskey pct debentures due said the debentures will continue converted into yankee preferred the remainder the price will one mln dlr note eskey yankee said the sale will result loss mln dlrs reuter '\n",
            " ' urges surplus nations boost growth leading industrial nations will reviewing the paris agreement stabilize exchange rates foster increased worldwide growth and reduce trade imbalances but the thinks the accord has been successful far senior treasury official said the paris accord will reviewed this meeting has been successful and continues succesfull senior treasury official told reporters briefing ahead this week semiannual imf and world bank meetings also said the was looking west germany and japan bolster their economic growth the official said both surplus countries like west germany and japan and deficit countries like the agreed play role bringing about more balanced economic growth reaffirmed the would press ahead with efforts reduce its budget deficit resist protectionism and boost competitiveness the official also said that expected trade issues like the dispute between the and japan over microchips included the discussions the official made direct comment the content schedule forthcoming group five and group seven discussions said that industrial countries are concerned that the large external imbalances remain threat the international monetary system added that the meetings will also provide opportunity discuss economic policy coordination efforts the official said indicators would used measure policy objectives industrial countries and their economic projections they would also used assess progress policy goals asked whether the was proposing new initiative regarding the indicators the official said the issue would reported the venice summit june monetary sources said the proposal envisages using the indicators make policy coordination agreements like the paris accord more binding reuter ']\n",
            "['alaska air group inc alk qtly dividend shr four cts four cts prior qtr pay may five record april reuter '\n",
            " 'thermo process tpsi acquisition terminated thermo process systems inc said its proposed acquisition the surface combustion division privately held midland ross corp has been terminated because mutually satisfactory terms could not established reuter '\n",
            " 'shared medical systems corp smed sets payout qtly div cts cts prior pay april record march reuter ']\n",
            "[0 1 2]\n",
            "[0 1 0]\n",
            "length of X_train:  4341\n",
            "length of X_test:  1086\n",
            "length of y_train:  4341\n",
            "length of y_test:  1086\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g6XckaAxhWhq",
        "outputId": "d23a6b3d-4187-4d6e-a6bc-81af1507c09a"
      },
      "source": [
        "# Split valtest set in to validation and test set\r\n",
        "X_val, X_test, y_val, y_test = train_test_split(X_valtest, \r\n",
        "                                                  y_valtest, \r\n",
        "                                                  test_size=0.20, \r\n",
        "                                                  random_state=43)\r\n",
        "\r\n",
        "print('length of X_val: ', len(X_val))\r\n",
        "print('length of X_test: ', len(X_test))\r\n",
        "print('length of y_val: ', len(y_val))\r\n",
        "print('length of y_test: ', len(y_test))\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "length of X_val:  868\n",
            "length of X_test:  218\n",
            "length of y_val:  868\n",
            "length of y_test:  218\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lEog6vyxQG4v"
      },
      "source": [
        "BertTokenizer and Encoding the Data\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aWvk5tYuYVnD"
      },
      "source": [
        "def convert_example_to_feature(sentence):\r\n",
        "  \r\n",
        "  # combine step for tokenization, WordPiece vector mapping, adding special tokens as well as truncating reviews longer than the max length\r\n",
        "  \r\n",
        "  return tokenizer.encode_plus(sentence, \r\n",
        "                add_special_tokens = True, # add [CLS], [SEP]\r\n",
        "                max_length = max_length, # max length of the text that can go to BERT\r\n",
        "                pad_to_max_length = True, # add [PAD] tokens\r\n",
        "                return_attention_mask = True, # add attention mask to not focus on pad tokens\r\n",
        "              )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NTsNWRORanF3"
      },
      "source": [
        "# map to the expected input to TFBertForSequenceClassification\r\n",
        "def map_example_to_dict(input_ids, attention_masks, token_type_ids, label=None):\r\n",
        "  return {\r\n",
        "      \"input_ids\": input_ids,\r\n",
        "      \"token_type_ids\": token_type_ids,\r\n",
        "      \"attention_mask\": attention_masks,\r\n",
        "  }, label\r\n",
        "\r\n",
        "def encode_examples(s, l, isTestSet=0 ):\r\n",
        "\r\n",
        "  # prepare list, so that we can build up final TensorFlow dataset from slices.\r\n",
        "  input_ids_list = []\r\n",
        "  token_type_ids_list = []\r\n",
        "  attention_mask_list = []\r\n",
        "  label_list = []\r\n",
        "  \r\n",
        "  if(isTestSet): #Do not append lables, because during model prediction we should not pass labels/target values\r\n",
        "      for sentence, label in zip(s, l):\r\n",
        "         bert_input = convert_example_to_feature(sentence)\r\n",
        "  \r\n",
        "         input_ids_list.append(bert_input['input_ids'])\r\n",
        "         token_type_ids_list.append(bert_input['token_type_ids'])\r\n",
        "         attention_mask_list.append(bert_input['attention_mask'])\r\n",
        "         #label_list.append([label])\r\n",
        "      return tf.data.Dataset.from_tensor_slices((input_ids_list, attention_mask_list, token_type_ids_list)).map(map_example_to_dict)\r\n",
        "  else: #during model training lables are required.\r\n",
        "      for sentence, label in zip(s, l):\r\n",
        "         bert_input = convert_example_to_feature(sentence)\r\n",
        "  \r\n",
        "         input_ids_list.append(bert_input['input_ids'])\r\n",
        "         token_type_ids_list.append(bert_input['token_type_ids'])\r\n",
        "         attention_mask_list.append(bert_input['attention_mask'])\r\n",
        "         label_list.append([label])\r\n",
        "      return tf.data.Dataset.from_tensor_slices((input_ids_list, attention_mask_list, token_type_ids_list, label_list)).map(map_example_to_dict)\r\n",
        "\r\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6-I2UXzda6pX",
        "outputId": "96b360ef-4993-42e9-c5f6-8fb5d64a04b4"
      },
      "source": [
        "# train dataset\r\n",
        "ds_train_encoded = encode_examples(X_train,y_train).shuffle(4341).batch(batch_size)\r\n",
        "\r\n",
        "#Validation dataset\r\n",
        "ds_val_encoded = encode_examples(X_val, y_val).batch(batch_size)\r\n",
        "\r\n",
        "# test dataset\r\n",
        "ds_test_encoded = encode_examples(X_test, y_test,1).batch(batch_size)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/transformers/tokenization_utils_base.py:2155: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Azr4pFH3IKFc"
      },
      "source": [
        "Below you can see the token, segment and positional embeddings of BERT. Shape of these are (6,200), where 6 is the size of the batch & 200 is the length of each sentence we specified.\r\n",
        "\r\n",
        "If sentence length is smaller than 200 it will be padded with 0s. If length is bigger, sentence will be truncated."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fRqLz5Crdozk",
        "outputId": "e370fe96-2906-4102-e7de-402418ed3373"
      },
      "source": [
        "print(type(ds_test_encoded))\r\n",
        "for sentence, label in ds_train_encoded.take(1):\r\n",
        "    print('sentence', sentence, label)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'tensorflow.python.data.ops.dataset_ops.BatchDataset'>\n",
            "sentence {'input_ids': <tf.Tensor: shape=(6, 200), dtype=int32, numpy=\n",
            "array([[  101,  2900, 23439, ...,     0,     0,     0],\n",
            "       [  101, 22714, 17180, ...,     0,     0,     0],\n",
            "       [  101,  9587,  3406, ...,     0,     0,     0],\n",
            "       [  101, 23876, 15726, ...,  4495,  6848,   102],\n",
            "       [  101, 23060, 24163, ...,     0,     0,     0],\n",
            "       [  101,  8174,  2015, ...,     0,     0,     0]], dtype=int32)>, 'token_type_ids': <tf.Tensor: shape=(6, 200), dtype=int32, numpy=\n",
            "array([[0, 0, 0, ..., 0, 0, 0],\n",
            "       [0, 0, 0, ..., 0, 0, 0],\n",
            "       [0, 0, 0, ..., 0, 0, 0],\n",
            "       [0, 0, 0, ..., 0, 0, 0],\n",
            "       [0, 0, 0, ..., 0, 0, 0],\n",
            "       [0, 0, 0, ..., 0, 0, 0]], dtype=int32)>, 'attention_mask': <tf.Tensor: shape=(6, 200), dtype=int32, numpy=\n",
            "array([[1, 1, 1, ..., 0, 0, 0],\n",
            "       [1, 1, 1, ..., 0, 0, 0],\n",
            "       [1, 1, 1, ..., 0, 0, 0],\n",
            "       [1, 1, 1, ..., 1, 1, 1],\n",
            "       [1, 1, 1, ..., 0, 0, 0],\n",
            "       [1, 1, 1, ..., 0, 0, 0]], dtype=int32)>} tf.Tensor(\n",
            "[[2]\n",
            " [1]\n",
            " [0]\n",
            " [2]\n",
            " [0]\n",
            " [5]], shape=(6, 1), dtype=int32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qek5SgvBf--G"
      },
      "source": [
        "**BERT Model Intialization**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GAnDhBvEgFyc"
      },
      "source": [
        "We will use already pretrained TensorFlow models from transformers models. You can just import them from the library and call from_pretrained and you will be able to use them"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c-0ZeeDqf1VC",
        "outputId": "dded4c36-bab7-4981-c84c-f7555841afb4"
      },
      "source": [
        "from transformers import TFBertForSequenceClassification\r\n",
        "import tensorflow as tf\r\n",
        "\r\n",
        "# recommended learning rate for Adam 5e-5, 3e-5, 2e-5\r\n",
        "learning_rate = 2e-5\r\n",
        "\r\n",
        "# we will do just 1 epoch for illustration, though multiple epochs might be better as long as we will not overfit the model\r\n",
        "number_of_epochs = 1\r\n",
        "\r\n",
        "# model initialization. For multi-class classification we must specify number of categories to classify the sentences/documents, 8 in our case.\r\n",
        "model = TFBertForSequenceClassification.from_pretrained('bert-base-uncased',num_labels=8)\r\n",
        "\r\n",
        "# choosing Adam optimizer\r\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate, epsilon=1e-08)\r\n",
        "\r\n",
        "# we can use sparce categorical cross entropy and accuracy\r\n",
        "loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\r\n",
        "metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy')\r\n",
        "\r\n",
        "model.compile(optimizer=optimizer, loss=loss, metrics=[metric])\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "All model checkpoint layers were used when initializing TFBertForSequenceClassification.\n",
            "\n",
            "Some layers of TFBertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-SzCNrWlr0nJ"
      },
      "source": [
        "**Train Bert Model**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2HhNZ7Q2gppj",
        "outputId": "992cda68-b2cb-4e07-d562-d5fca388a0c0"
      },
      "source": [
        "model.fit(ds_train_encoded, epochs=number_of_epochs, validation_data=ds_val_encoded)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n",
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n",
            "724/724 [==============================] - ETA: 0s - loss: 0.6444 - accuracy: 0.8100WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n",
            "724/724 [==============================] - 224s 296ms/step - loss: 0.6440 - accuracy: 0.8102 - val_loss: 0.1232 - val_accuracy: 0.9712\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7fc5495f8240>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jY8N_fgSr5Ej"
      },
      "source": [
        "**Classify/Predict unseen test sentences**\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qQRoiLJ7r8Hu",
        "outputId": "fac25ade-8c51-4ced-9466-32bf8e3b5f59"
      },
      "source": [
        "test_pred = model.predict(ds_test_encoded, verbose=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n",
            "37/37 [==============================] - 4s 84ms/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9yTqS9ov20mr",
        "outputId": "ef03fe4f-7ec2-4580-d6fe-e45c87cc80d7"
      },
      "source": [
        "test_pred"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TFSequenceClassifierOutput([('logits',\n",
              "                             array([[ 0.10074364,  5.658406  , -1.0375811 , ..., -1.1489806 ,\n",
              "                                     -0.5750521 , -0.95126194],\n",
              "                                    [ 6.435535  , -0.03667761, -1.3546342 , ..., -1.2319204 ,\n",
              "                                     -0.7304846 , -1.1815776 ],\n",
              "                                    [ 6.4013224 , -0.0300498 , -1.380059  , ..., -1.242933  ,\n",
              "                                     -0.7095235 , -1.166215  ],\n",
              "                                    ...,\n",
              "                                    [ 6.3813424 ,  0.06080941, -1.3787117 , ..., -1.2683374 ,\n",
              "                                     -0.6816839 , -1.1932096 ],\n",
              "                                    [-1.2148149 , -1.3147076 , -0.33345002, ..., -0.8928549 ,\n",
              "                                      0.80356324,  3.9285421 ],\n",
              "                                    [ 6.319223  ,  0.11768577, -1.4491475 , ..., -1.4584801 ,\n",
              "                                     -0.42460883, -1.0240362 ]], dtype=float32))])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 212
        },
        "id": "Grn5-YLCoey4",
        "outputId": "448d0d42-8bd7-4fe8-94c2-9fe6929d8a6a"
      },
      "source": [
        "op = pd.DataFrame(test_pred[0])\r\n",
        "print(op.shape)\r\n",
        "op.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(218, 8)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.100744</td>\n",
              "      <td>5.658406</td>\n",
              "      <td>-1.037581</td>\n",
              "      <td>-0.421502</td>\n",
              "      <td>-1.073307</td>\n",
              "      <td>-1.148981</td>\n",
              "      <td>-0.575052</td>\n",
              "      <td>-0.951262</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>6.435535</td>\n",
              "      <td>-0.036678</td>\n",
              "      <td>-1.354634</td>\n",
              "      <td>-1.188579</td>\n",
              "      <td>-0.985788</td>\n",
              "      <td>-1.231920</td>\n",
              "      <td>-0.730485</td>\n",
              "      <td>-1.181578</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>6.401322</td>\n",
              "      <td>-0.030050</td>\n",
              "      <td>-1.380059</td>\n",
              "      <td>-1.206783</td>\n",
              "      <td>-1.003658</td>\n",
              "      <td>-1.242933</td>\n",
              "      <td>-0.709523</td>\n",
              "      <td>-1.166215</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>6.377688</td>\n",
              "      <td>0.042810</td>\n",
              "      <td>-1.385330</td>\n",
              "      <td>-1.274339</td>\n",
              "      <td>-1.017310</td>\n",
              "      <td>-1.256177</td>\n",
              "      <td>-0.688236</td>\n",
              "      <td>-1.156304</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>6.398369</td>\n",
              "      <td>0.099404</td>\n",
              "      <td>-1.407508</td>\n",
              "      <td>-1.250317</td>\n",
              "      <td>-1.019872</td>\n",
              "      <td>-1.237434</td>\n",
              "      <td>-0.670231</td>\n",
              "      <td>-1.159171</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "          0         1         2  ...         5         6         7\n",
              "0  0.100744  5.658406 -1.037581  ... -1.148981 -0.575052 -0.951262\n",
              "1  6.435535 -0.036678 -1.354634  ... -1.231920 -0.730485 -1.181578\n",
              "2  6.401322 -0.030050 -1.380059  ... -1.242933 -0.709523 -1.166215\n",
              "3  6.377688  0.042810 -1.385330  ... -1.256177 -0.688236 -1.156304\n",
              "4  6.398369  0.099404 -1.407508  ... -1.237434 -0.670231 -1.159171\n",
              "\n",
              "[5 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xPieka9sKJlA"
      },
      "source": [
        "Extract Predicted Label for each sentence in the test data set."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AZN-O77BsSNr",
        "outputId": "ce8f61d0-3898-4eef-a974-be466cd56b6d"
      },
      "source": [
        "#Get predicted class for each sentence in the test set\r\n",
        "predicted_label = op.idxmax(axis=1)\r\n",
        "predicted_label"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0      1\n",
              "1      0\n",
              "2      0\n",
              "3      0\n",
              "4      0\n",
              "      ..\n",
              "213    0\n",
              "214    1\n",
              "215    0\n",
              "216    7\n",
              "217    0\n",
              "Length: 218, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L_WId4doKZJn"
      },
      "source": [
        "Create a DataFrame that shows sentence, actual label and, predicted label for test data set."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "id": "h886Y0uvtcLy",
        "outputId": "71de07e4-4cac-4d19-f97d-684dc9bb2faa"
      },
      "source": [
        "predicted = pd.DataFrame(predicted_label)\r\n",
        "actual = pd.DataFrame(y_test)\r\n",
        "sentence = pd.DataFrame(X_test)\r\n",
        "testset_results = pd.concat([sentence, actual, predicted],axis=1)\r\n",
        "testset_results.columns = ['sentence', 'actual', 'predicted']\r\n",
        "testset_results.tail()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence</th>\n",
              "      <th>actual</th>\n",
              "      <th>predicted</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>213</th>\n",
              "      <td>quest medical inc qmed qtr loss shr loss six c...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>214</th>\n",
              "      <td>usair has comment twa twa offer usair group in...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>215</th>\n",
              "      <td>peps boys manny moe and jack pby set payout qt...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>216</th>\n",
              "      <td>money market given mln stg late assistance th...</td>\n",
              "      <td>7</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>217</th>\n",
              "      <td>primebank pmbk sets pct stock dividend primeba...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                              sentence  actual  predicted\n",
              "213  quest medical inc qmed qtr loss shr loss six c...       0          0\n",
              "214  usair has comment twa twa offer usair group in...       1          1\n",
              "215  peps boys manny moe and jack pby set payout qt...       0          0\n",
              "216   money market given mln stg late assistance th...       7          7\n",
              "217  primebank pmbk sets pct stock dividend primeba...       0          0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DNiQ_9jMv-kF"
      },
      "source": [
        "**Performance Metrics**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JgWTNvNwwFN0",
        "outputId": "487c6c95-78f0-4431-8ee1-63919c544983"
      },
      "source": [
        "from sklearn.metrics import accuracy_score,confusion_matrix, classification_report\r\n",
        "\r\n",
        "ascore = accuracy_score(testset_results.actual, testset_results.predicted)\r\n",
        "print(\"Bert Accuracy Score: \", ascore)\r\n",
        "print(\"Confusion Matrix of BERT Classifier output: \")\r\n",
        "confusion_matrix(testset_results.actual, testset_results.predicted)\r\n",
        "print(\"Classification Metrics: \")\r\n",
        "print(classification_report(testset_results.actual, testset_results.predicted))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Bert Accuracy Score:  0.9724770642201835\n",
            "Confusion Matrix of BERT Classifier output: \n",
            "Classification Metrics: \n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.97      1.00      0.99       116\n",
            "           1       0.97      0.95      0.96        59\n",
            "           2       1.00      1.00      1.00        15\n",
            "           3       1.00      0.75      0.86         4\n",
            "           5       1.00      0.71      0.83         7\n",
            "           6       1.00      1.00      1.00         7\n",
            "           7       0.91      1.00      0.95        10\n",
            "\n",
            "    accuracy                           0.97       218\n",
            "   macro avg       0.98      0.92      0.94       218\n",
            "weighted avg       0.97      0.97      0.97       218\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}