tutorials/Certification_Trainings/Healthcare/clinical_text_classification/3.Clinical_Longformer_vs_BertSentence_&_USE.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "id": "719de9b1-b9cd-417f-8ef6-6c0c2290282b",
      "metadata": {
        "id": "719de9b1-b9cd-417f-8ef6-6c0c2290282b"
      },
      "source": [
        "![JohnSnowLabs](https://nlp.johnsnowlabs.com/assets/images/logo.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/clinical_text_classification/3.Clinical_Longformer_vs_BertSentence_&_USE.ipynb)"
      ],
      "metadata": {
        "id": "R34vIiledKVo"
      },
      "id": "R34vIiledKVo"
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Colab Setup"
      ],
      "metadata": {
        "id": "A-oK9vkadKDz"
      },
      "id": "A-oK9vkadKDz"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "99b171c3-4100-4917-a41a-164ff7801dc9",
      "metadata": {
        "id": "99b171c3-4100-4917-a41a-164ff7801dc9"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "import os\n",
        "from google.colab import files\n",
        "\n",
        "license_keys = files.upload()\n",
        "\n",
        "with open(list(license_keys.keys())[0]) as f:\n",
        "    license_keys = json.load(f)\n",
        "\n",
        "# Defining license key-value pairs as local variables\n",
        "locals().update(license_keys)\n",
        "\n",
        "# Adding license key-value pairs to environment variables\n",
        "\n",
        "os.environ.update(license_keys)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "id": "7680c7db-0c83-4761-bd58-e5b951604ef2",
      "metadata": {
        "id": "7680c7db-0c83-4761-bd58-e5b951604ef2"
      },
      "outputs": [],
      "source": [
        "# Installing pyspark and spark-nlp\n",
        "! pip install --upgrade -q pyspark==3.1.2 spark-nlp==$PUBLIC_VERSION\n",
        "\n",
        "# Installing Spark NLP Healthcare\n",
        "! pip install --upgrade -q spark-nlp-jsl==$JSL_VERSION  --extra-index-url https://pypi.johnsnowlabs.com/$SECRET\n",
        "\n",
        "# Installing Spark NLP Display Library for visualization\n",
        "! pip install -q spark-nlp-display"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "id": "7ca73d53-4fcb-4734-ae73-670b5048a21b",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 271
        },
        "id": "7ca73d53-4fcb-4734-ae73-670b5048a21b",
        "outputId": "5719a5e8-4150-49d2-9b6b-280d09fd817c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Spark NLP Version : 3.4.0\n",
            "Spark NLP_JSL Version : 3.4.0\n",
            "Warning::Spark Session already created, some configs may not take.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "            <div>\n",
              "                <p><b>SparkSession - in-memory</b></p>\n",
              "                \n",
              "        <div>\n",
              "            <p><b>SparkContext</b></p>\n",
              "\n",
              "            <p><a href=\"http://6cf5a203642d:4040\">Spark UI</a></p>\n",
              "\n",
              "            <dl>\n",
              "              <dt>Version</dt>\n",
              "                <dd><code>v3.1.2</code></dd>\n",
              "              <dt>Master</dt>\n",
              "                <dd><code>local[*]</code></dd>\n",
              "              <dt>AppName</dt>\n",
              "                <dd><code>Spark NLP Licensed</code></dd>\n",
              "            </dl>\n",
              "        </div>\n",
              "        \n",
              "            </div>\n",
              "        "
            ],
            "text/plain": [
              "<pyspark.sql.session.SparkSession at 0x7fbf223b16d0>"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ],
      "source": [
        "from pyspark.ml import Pipeline,PipelineModel\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql import functions as F\n",
        "\n",
        "from sparknlp.annotator import *\n",
        "from sparknlp_jsl.annotator import *\n",
        "from sparknlp.base import *\n",
        "import sparknlp_jsl\n",
        "import sparknlp\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "\n",
        "params = {\"spark.driver.memory\":\"16G\", \n",
        "          \"spark.kryoserializer.buffer.max\":\"2000M\", \n",
        "          \"spark.driver.maxResultSize\":\"2000M\"} \n",
        "\n",
        "print (\"Spark NLP Version :\", sparknlp.version())\n",
        "print (\"Spark NLP_JSL Version :\", sparknlp_jsl.version())\n",
        "\n",
        "spark = sparknlp_jsl.start(license_keys['SECRET'],params=params)\n",
        "\n",
        "spark"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# if you want to start the session with custom params as in start function above\n",
        "from pyspark.sql import SparkSession\n",
        "\n",
        "def start(SECRET):\n",
        "    builder = SparkSession.builder \\\n",
        "        .appName(\"Spark NLP Licensed\") \\\n",
        "        .master(\"local[*]\") \\\n",
        "        .config(\"spark.driver.memory\", \"16G\") \\\n",
        "        .config(\"spark.serializer\", \"org.apache.spark.serializer.KryoSerializer\") \\\n",
        "        .config(\"spark.kryoserializer.buffer.max\", \"2000M\") \\\n",
        "        .config(\"spark.jars.packages\", \"com.johnsnowlabs.nlp:spark-nlp_2.12:\"+PUBLIC_VERSION) \\\n",
        "        .config(\"spark.jars\", \"https://pypi.johnsnowlabs.com/\"+SECRET+\"/spark-nlp-jsl-\"+JSL_VERSION+\".jar\")\n",
        "      \n",
        "    return builder.getOrCreate()\n",
        "\n",
        "#spark = start(SECRET)"
      ],
      "metadata": {
        "id": "JoRhmw_35F8j"
      },
      "id": "JoRhmw_35F8j",
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Download Mtsamples Dataset"
      ],
      "metadata": {
        "id": "RKA4wtpDdS4T"
      },
      "id": "RKA4wtpDdS4T"
    },
    {
      "cell_type": "code",
      "source": [
        "!wget -q https://raw.githubusercontent.com/JohnSnowLabs/spark-nlp-workshop/master/tutorials/Certification_Trainings/Healthcare/data/mtsamples_classifier.csv"
      ],
      "metadata": {
        "id": "CHPqy3NeZMWW"
      },
      "id": "CHPqy3NeZMWW",
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "id": "44eb4f5f-9065-4512-aad6-2afc89f74220",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "44eb4f5f-9065-4512-aad6-2afc89f74220",
        "outputId": "1af5a773-c7a4-4f03-ef78-f0fe58b90ab3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------------+--------------------+\n",
            "|        category|                text|\n",
            "+----------------+--------------------+\n",
            "|Gastroenterology| PROCEDURES PERFO...|\n",
            "|Gastroenterology| OPERATION 1. Ivo...|\n",
            "|Gastroenterology| PREOPERATIVE DIA...|\n",
            "|Gastroenterology| PROCEDURE: Colon...|\n",
            "|Gastroenterology| PREOPERATIVE DIA...|\n",
            "|Gastroenterology| PREOPERATIVE DIA...|\n",
            "|Gastroenterology| The patient was ...|\n",
            "|Gastroenterology| PREOPERATIVE DIA...|\n",
            "|Gastroenterology| PREOPERATIVE DIA...|\n",
            "|Gastroenterology| HISTORY OF PRESE...|\n",
            "|Gastroenterology| PREOPERATIVE DIA...|\n",
            "|Gastroenterology| PREOPERATIVE DIA...|\n",
            "|Gastroenterology| PROCEDURE IN DET...|\n",
            "|Gastroenterology| PREPROCEDURE DIA...|\n",
            "|Gastroenterology| The patient is a...|\n",
            "|Gastroenterology| DIAGNOSIS ON ADM...|\n",
            "|Gastroenterology| EXAM: CT abdomen...|\n",
            "|Gastroenterology| Sample Doctor, M...|\n",
            "|Gastroenterology| CHIEF COMPLAINT:...|\n",
            "|Gastroenterology| PREOPERATIVE DIA...|\n",
            "+----------------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "spark_df = spark.read.csv(\"mtsamples_classifier.csv\", header = True)\n",
        "\n",
        "spark_df.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f0886fb5-fd9d-4199-b142-637cad880866",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f0886fb5-fd9d-4199-b142-637cad880866",
        "outputId": "f9647237-42b4-4121-e9e6-28262595cd0c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "638"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ],
      "source": [
        "spark_df.count()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f99d7e2f-e3b9-42a9-8f37-71bf64a1cd31",
      "metadata": {
        "id": "f99d7e2f-e3b9-42a9-8f37-71bf64a1cd31"
      },
      "outputs": [],
      "source": [
        "spark_df = spark_df.filter((spark_df.text != \"\"))\n",
        "# None values in the text column of data were removed from the data --> 8 rows"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "0c3aa1ca-ea85-4eca-aa92-b10d97e9d7a5",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0c3aa1ca-ea85-4eca-aa92-b10d97e9d7a5",
        "outputId": "02ade455-06a2-4cfe-ac20-85e9e6751203"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "630"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ],
      "source": [
        "spark_df.count()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "241bd286-350c-421e-92f4-a80164ed2dc1",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "241bd286-350c-421e-92f4-a80164ed2dc1",
        "outputId": "cd3113c4-20a0-4285-ede2-1c1a9f80a688"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------------+-----+\n",
            "|        category|count|\n",
            "+----------------+-----+\n",
            "|         Urology|  110|\n",
            "|       Neurology|  141|\n",
            "|      Orthopedic|  222|\n",
            "|Gastroenterology|  157|\n",
            "+----------------+-----+\n",
            "\n"
          ]
        }
      ],
      "source": [
        "spark_df.groupBy(\"category\").count().show()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "abc5a754-de2d-41e1-a571-4047bd249d1c",
      "metadata": {
        "id": "abc5a754-de2d-41e1-a571-4047bd249d1c"
      },
      "source": [
        "**I tried the following values for all models**\n",
        "- **setLr :** 0.001, 0.003, 0.005, 0.0001, 0.0003, 0.0005\n",
        "- **setMaxEpochs :** 16, 32, 64, 128\n",
        "- **setBatchSize :** 8, 10, 16, 32\n",
        "- **setDropout :** 0.2, 0.3, 0.4, 0.5"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "db12fe42-90b6-434d-852a-ea86cad9a963",
      "metadata": {
        "id": "db12fe42-90b6-434d-852a-ea86cad9a963"
      },
      "outputs": [],
      "source": [
        "(trainingData, testData) = spark_df.randomSplit([0.8, 0.2], seed = 42)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "ffca7d5b-02db-477b-90d6-af768e0f270a",
      "metadata": {
        "id": "ffca7d5b-02db-477b-90d6-af768e0f270a"
      },
      "source": [
        "## Universal Sentence Encoder"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3faaec20-0d92-489d-97d9-346554e54a49",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3faaec20-0d92-489d-97d9-346554e54a49",
        "outputId": "d7045c96-37bc-45da-a439-398d760a29ce"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tfhub_use download started this may take some time.\n",
            "Approximate size to download 923.7 MB\n",
            "[OK!]\n"
          ]
        }
      ],
      "source": [
        "document_assembler = DocumentAssembler()\\\n",
        "        .setInputCol(\"text\")\\\n",
        "        .setOutputCol(\"document\")\n",
        "\n",
        "use = UniversalSentenceEncoder.pretrained()\\\n",
        "        .setInputCols(\"document\")\\\n",
        "        .setOutputCol(\"sentence_embeddings\")\n",
        "\n",
        "classifier_dl = ClassifierDLApproach()\\\n",
        "        .setInputCols([\"sentence_embeddings\"])\\\n",
        "        .setOutputCol(\"class\")\\\n",
        "        .setLabelColumn(\"category\")\\\n",
        "        .setBatchSize(8)\\\n",
        "        .setMaxEpochs(64)\\\n",
        "        .setLr(0.003)\\\n",
        "        .setDropout(0.3)\\\n",
        "        .setEnableOutputLogs(True)\\\n",
        "\n",
        "use_clf_pipeline = Pipeline(\n",
        "    stages = [\n",
        "        document_assembler,\n",
        "        use,\n",
        "        classifier_dl\n",
        "    ])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "34e5a446-8f0e-4280-a3a9-16c337d0faa4",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "34e5a446-8f0e-4280-a3a9-16c337d0faa4",
        "outputId": "fae7feb5-c631-47ad-adb6-5e23f74e7536"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 172 ms, sys: 21.1 ms, total: 193 ms\n",
            "Wall time: 29.7 s\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "use_clf_pipelineModel = use_clf_pipeline.fit(trainingData)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!cd ~/annotator_logs/ && ls -lt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oWHWvRxh6Gre",
        "outputId": "2a95d8c5-1b94-4391-942e-c7789ef00a04"
      },
      "id": "oWHWvRxh6Gre",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "total 32\n",
            "-rw-r--r-- 1 root root 4507 Feb 15 21:53 ClassifierDLApproach_41c1c26d040d.log\n",
            "-rw-r--r-- 1 root root 9020 Feb 15 21:03 ClassifierDLApproach_726271d783f5.log\n",
            "-rw-r--r-- 1 root root 9075 Feb 15 20:52 ClassifierDLApproach_9571e7315f7f.log\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!cat ~/annotator_logs/ClassifierDLApproach_8442e46ada47.log"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oju51r-06H_q",
        "outputId": "bd2b1a27-ab69-44f1-853f-cb6eb75e0c55"
      },
      "id": "oju51r-06H_q",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training started - epochs: 64 - learning_rate: 0.003 - batch_size: 8 - training_examples: 503 - classes: 4\n",
            "Epoch 0/64 - 0.52s - loss: 74.91817 - acc: 0.51180875 - batches: 63\n",
            "Epoch 1/64 - 0.36s - loss: 68.47723 - acc: 0.6569701 - batches: 63\n",
            "Epoch 2/64 - 0.34s - loss: 62.247604 - acc: 0.74769586 - batches: 63\n",
            "Epoch 3/64 - 0.34s - loss: 58.552536 - acc: 0.82430875 - batches: 63\n",
            "Epoch 4/64 - 0.35s - loss: 55.85074 - acc: 0.85656685 - batches: 63\n",
            "Epoch 5/64 - 0.34s - loss: 54.345898 - acc: 0.8666475 - batches: 63\n",
            "Epoch 6/64 - 0.35s - loss: 52.835968 - acc: 0.890841 - batches: 63\n",
            "Epoch 7/64 - 0.35s - loss: 51.885544 - acc: 0.9133065 - batches: 63\n",
            "Epoch 8/64 - 0.35s - loss: 51.30577 - acc: 0.91935486 - batches: 63\n",
            "Epoch 9/64 - 0.34s - loss: 50.97224 - acc: 0.92741936 - batches: 63\n",
            "Epoch 10/64 - 0.36s - loss: 50.771362 - acc: 0.93346775 - batches: 63\n",
            "Epoch 11/64 - 0.35s - loss: 50.639965 - acc: 0.93346775 - batches: 63\n",
            "Epoch 12/64 - 0.34s - loss: 50.550507 - acc: 0.93346775 - batches: 63\n",
            "Epoch 13/64 - 0.35s - loss: 50.48486 - acc: 0.9354839 - batches: 63\n",
            "Epoch 14/64 - 0.34s - loss: 50.434525 - acc: 0.9395161 - batches: 63\n",
            "Epoch 15/64 - 0.35s - loss: 50.392895 - acc: 0.9395161 - batches: 63\n",
            "Epoch 16/64 - 0.35s - loss: 50.35681 - acc: 0.94153225 - batches: 63\n",
            "Epoch 17/64 - 0.35s - loss: 50.327034 - acc: 0.94153225 - batches: 63\n",
            "Epoch 18/64 - 0.35s - loss: 50.30075 - acc: 0.9435484 - batches: 63\n",
            "Epoch 19/64 - 0.35s - loss: 50.277557 - acc: 0.9435484 - batches: 63\n",
            "Epoch 20/64 - 0.35s - loss: 50.257214 - acc: 0.9455645 - batches: 63\n",
            "Epoch 21/64 - 0.36s - loss: 50.23933 - acc: 0.9455645 - batches: 63\n",
            "Epoch 22/64 - 0.36s - loss: 50.223557 - acc: 0.9455645 - batches: 63\n",
            "Epoch 23/64 - 0.35s - loss: 50.208935 - acc: 0.9455645 - batches: 63\n",
            "Epoch 24/64 - 0.36s - loss: 50.196255 - acc: 0.9455645 - batches: 63\n",
            "Epoch 25/64 - 0.35s - loss: 50.184616 - acc: 0.9455645 - batches: 63\n",
            "Epoch 26/64 - 0.35s - loss: 50.174046 - acc: 0.9455645 - batches: 63\n",
            "Epoch 27/64 - 0.36s - loss: 50.16442 - acc: 0.9516129 - batches: 63\n",
            "Epoch 28/64 - 0.38s - loss: 50.15556 - acc: 0.9516129 - batches: 63\n",
            "Epoch 29/64 - 0.35s - loss: 50.147144 - acc: 0.9516129 - batches: 63\n",
            "Epoch 30/64 - 0.37s - loss: 50.139664 - acc: 0.9516129 - batches: 63\n",
            "Epoch 31/64 - 0.36s - loss: 50.132507 - acc: 0.9516129 - batches: 63\n",
            "Epoch 32/64 - 0.34s - loss: 50.12583 - acc: 0.9516129 - batches: 63\n",
            "Epoch 33/64 - 0.35s - loss: 50.11969 - acc: 0.9516129 - batches: 63\n",
            "Epoch 34/64 - 0.35s - loss: 50.11373 - acc: 0.9516129 - batches: 63\n",
            "Epoch 35/64 - 0.37s - loss: 50.10817 - acc: 0.9516129 - batches: 63\n",
            "Epoch 36/64 - 0.37s - loss: 50.103058 - acc: 0.9516129 - batches: 63\n",
            "Epoch 37/64 - 0.35s - loss: 50.09819 - acc: 0.9516129 - batches: 63\n",
            "Epoch 38/64 - 0.34s - loss: 50.093704 - acc: 0.9516129 - batches: 63\n",
            "Epoch 39/64 - 0.35s - loss: 50.089264 - acc: 0.9516129 - batches: 63\n",
            "Epoch 40/64 - 0.36s - loss: 50.08498 - acc: 0.9516129 - batches: 63\n",
            "Epoch 41/64 - 0.35s - loss: 50.081055 - acc: 0.9516129 - batches: 63\n",
            "Epoch 42/64 - 0.37s - loss: 50.076996 - acc: 0.9516129 - batches: 63\n",
            "Epoch 43/64 - 0.36s - loss: 50.07294 - acc: 0.9516129 - batches: 63\n",
            "Epoch 44/64 - 0.37s - loss: 50.069214 - acc: 0.9516129 - batches: 63\n",
            "Epoch 45/64 - 0.38s - loss: 50.065792 - acc: 0.9516129 - batches: 63\n",
            "Epoch 46/64 - 0.42s - loss: 50.062527 - acc: 0.9516129 - batches: 63\n",
            "Epoch 47/64 - 0.35s - loss: 50.05942 - acc: 0.9516129 - batches: 63\n",
            "Epoch 48/64 - 0.35s - loss: 50.056458 - acc: 0.953629 - batches: 63\n",
            "Epoch 49/64 - 0.35s - loss: 50.053654 - acc: 0.953629 - batches: 63\n",
            "Epoch 50/64 - 0.37s - loss: 50.050915 - acc: 0.953629 - batches: 63\n",
            "Epoch 51/64 - 0.35s - loss: 50.048576 - acc: 0.953629 - batches: 63\n",
            "Epoch 52/64 - 0.35s - loss: 50.0463 - acc: 0.953629 - batches: 63\n",
            "Epoch 53/64 - 0.36s - loss: 50.04384 - acc: 0.953629 - batches: 63\n",
            "Epoch 54/64 - 0.36s - loss: 50.04169 - acc: 0.953629 - batches: 63\n",
            "Epoch 55/64 - 0.34s - loss: 50.03969 - acc: 0.953629 - batches: 63\n",
            "Epoch 56/64 - 0.36s - loss: 50.03772 - acc: 0.953629 - batches: 63\n",
            "Epoch 57/64 - 0.35s - loss: 50.03586 - acc: 0.953629 - batches: 63\n",
            "Epoch 58/64 - 0.35s - loss: 50.03404 - acc: 0.953629 - batches: 63\n",
            "Epoch 59/64 - 0.36s - loss: 50.032303 - acc: 0.953629 - batches: 63\n",
            "Epoch 60/64 - 0.35s - loss: 50.030647 - acc: 0.953629 - batches: 63\n",
            "Epoch 61/64 - 0.35s - loss: 50.029083 - acc: 0.953629 - batches: 63\n",
            "Epoch 62/64 - 0.36s - loss: 50.027622 - acc: 0.953629 - batches: 63\n",
            "Epoch 63/64 - 0.36s - loss: 50.026188 - acc: 0.953629 - batches: 63\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "As you can see,  we achieved **95% accuracy score** in less than 1 min with no text preprocessing, which is usually the most time consuming and laborious step in any NLP modeling.We achieved **91% test set accuracy!**"
      ],
      "metadata": {
        "id": "GjPVIoELdhLo"
      },
      "id": "GjPVIoELdhLo"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "19373389-cf39-41cc-a02e-15a3a3fda155",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "19373389-cf39-41cc-a02e-15a3a3fda155",
        "outputId": "dcc90b0b-8d1c-4e2b-adb4-ad4a24140fba"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                  precision    recall  f1-score   support\n",
            "\n",
            "Gastroenterology       0.90      0.90      0.90        31\n",
            "       Neurology       0.89      0.89      0.89        28\n",
            "      Orthopedic       0.95      0.90      0.92        41\n",
            "         Urology       0.86      0.93      0.89        27\n",
            "\n",
            "        accuracy                           0.91       127\n",
            "       macro avg       0.90      0.91      0.90       127\n",
            "    weighted avg       0.91      0.91      0.91       127\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from sklearn.metrics import confusion_matrix, classification_report, accuracy_score\n",
        "\n",
        "preds = use_clf_pipelineModel.transform(testData)\n",
        "\n",
        "preds_df = preds.select(\"category\",\"text\",\"class.result\").toPandas()\n",
        "\n",
        "preds_df[\"result\"] = preds_df[\"result\"].apply(lambda x : x[0])\n",
        "\n",
        "print (classification_report(preds_df[\"category\"], preds_df[\"result\"]))"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "fb1f6f12-d01f-453e-a611-0885a2da99de",
      "metadata": {
        "id": "fb1f6f12-d01f-453e-a611-0885a2da99de"
      },
      "source": [
        "## Bert Sentence Embeddings(sent_biobert_clinical_base_cased)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b821fa51-5897-4717-8997-7c442ef55db2",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b821fa51-5897-4717-8997-7c442ef55db2",
        "outputId": "ef1a8c97-bf6d-4a7b-e616-534065e7f0b5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "sent_biobert_clinical_base_cased download started this may take some time.\n",
            "Approximate size to download 386.6 MB\n",
            "[OK!]\n"
          ]
        }
      ],
      "source": [
        "document_assembler = DocumentAssembler()\\\n",
        "        .setInputCol(\"text\")\\\n",
        "        .setOutputCol(\"document\")\n",
        "        \n",
        "bert_sent = BertSentenceEmbeddings.pretrained(\"sent_biobert_clinical_base_cased\")\\\n",
        "      .setInputCols([\"document\"])\\\n",
        "      .setOutputCol(\"sentence_embeddings\")\n",
        "\n",
        "classsifier_dl = ClassifierDLApproach()\\\n",
        "      .setInputCols([\"sentence_embeddings\"])\\\n",
        "      .setOutputCol(\"class\")\\\n",
        "      .setLabelColumn(\"category\")\\\n",
        "      .setMaxEpochs(128)\\\n",
        "      .setBatchSize(8)\\\n",
        "      .setEnableOutputLogs(True)\\\n",
        "      .setLr(0.0005)\\\n",
        "      .setDropout(0.2)\n",
        "\n",
        "bert_sent_clf_pipeline = Pipeline(\n",
        "    stages = [\n",
        "        document_assembler,\n",
        "        bert_sent,\n",
        "        classsifier_dl\n",
        "    ])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "dbabb107-2180-4dbe-881c-e7eb449e88c7",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dbabb107-2180-4dbe-881c-e7eb449e88c7",
        "outputId": "7136adcb-b7ef-4f6a-cb08-9d996c397256"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 2.65 s, sys: 296 ms, total: 2.95 s\n",
            "Wall time: 7min 47s\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "bert_sent_pipelineModel = bert_sent_clf_pipeline.fit(trainingData)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!cd ~/annotator_logs/ && ls -lt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CMeKkgNhByfN",
        "outputId": "69c72a35-0994-4893-bddf-8f7775cf2909"
      },
      "id": "CMeKkgNhByfN",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "total 24\n",
            "-rw-r--r-- 1 root root 9020 Feb 15 21:03 ClassifierDLApproach_726271d783f5.log\n",
            "-rw-r--r-- 1 root root 9075 Feb 15 20:52 ClassifierDLApproach_9571e7315f7f.log\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!cat ~/annotator_logs/ClassifierDLApproach_726271d783f5.log"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pgKtkJP_BxYM",
        "outputId": "ee7592b7-9de2-49e1-b8d0-290e4a9964db"
      },
      "id": "pgKtkJP_BxYM",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training started - epochs: 128 - learning_rate: 5.0E-4 - batch_size: 8 - training_examples: 528 - classes: 4\n",
            "Epoch 0/128 - 0.61s - loss: 86.71257 - acc: 0.39204547 - batches: 66\n",
            "Epoch 1/128 - 0.46s - loss: 77.47093 - acc: 0.6136364 - batches: 66\n",
            "Epoch 2/128 - 0.44s - loss: 70.96891 - acc: 0.6875 - batches: 66\n",
            "Epoch 3/128 - 0.46s - loss: 68.40075 - acc: 0.7140151 - batches: 66\n",
            "Epoch 4/128 - 0.48s - loss: 67.243576 - acc: 0.72159094 - batches: 66\n",
            "Epoch 5/128 - 0.45s - loss: 66.59227 - acc: 0.7310606 - batches: 66\n",
            "Epoch 6/128 - 0.45s - loss: 66.1715 - acc: 0.7405303 - batches: 66\n",
            "Epoch 7/128 - 0.44s - loss: 65.86946 - acc: 0.74810606 - batches: 66\n",
            "Epoch 8/128 - 0.66s - loss: 65.62933 - acc: 0.75 - batches: 66\n",
            "Epoch 9/128 - 0.46s - loss: 65.43666 - acc: 0.75189394 - batches: 66\n",
            "Epoch 10/128 - 0.46s - loss: 65.26919 - acc: 0.75189394 - batches: 66\n",
            "Epoch 11/128 - 0.44s - loss: 65.11844 - acc: 0.7556818 - batches: 66\n",
            "Epoch 12/128 - 0.46s - loss: 64.98228 - acc: 0.7613636 - batches: 66\n",
            "Epoch 13/128 - 0.43s - loss: 64.85689 - acc: 0.76325756 - batches: 66\n",
            "Epoch 14/128 - 0.49s - loss: 64.743225 - acc: 0.76325756 - batches: 66\n",
            "Epoch 15/128 - 0.43s - loss: 64.6389 - acc: 0.7651515 - batches: 66\n",
            "Epoch 16/128 - 0.43s - loss: 64.54027 - acc: 0.7708333 - batches: 66\n",
            "Epoch 17/128 - 0.45s - loss: 64.445244 - acc: 0.77272725 - batches: 66\n",
            "Epoch 18/128 - 0.52s - loss: 64.35155 - acc: 0.77272725 - batches: 66\n",
            "Epoch 19/128 - 0.52s - loss: 64.2556 - acc: 0.77272725 - batches: 66\n",
            "Epoch 20/128 - 0.48s - loss: 64.15211 - acc: 0.7746212 - batches: 66\n",
            "Epoch 21/128 - 0.44s - loss: 64.03403 - acc: 0.7746212 - batches: 66\n",
            "Epoch 22/128 - 0.44s - loss: 63.897648 - acc: 0.7765151 - batches: 66\n",
            "Epoch 23/128 - 0.45s - loss: 63.739372 - acc: 0.7765151 - batches: 66\n",
            "Epoch 24/128 - 0.42s - loss: 63.55913 - acc: 0.7765151 - batches: 66\n",
            "Epoch 25/128 - 0.45s - loss: 63.367435 - acc: 0.77840906 - batches: 66\n",
            "Epoch 26/128 - 0.45s - loss: 63.17929 - acc: 0.7935606 - batches: 66\n",
            "Epoch 27/128 - 0.44s - loss: 63.00413 - acc: 0.80492425 - batches: 66\n",
            "Epoch 28/128 - 0.43s - loss: 62.845886 - acc: 0.82007575 - batches: 66\n",
            "Epoch 29/128 - 0.42s - loss: 62.703213 - acc: 0.844697 - batches: 66\n",
            "Epoch 30/128 - 0.47s - loss: 62.572224 - acc: 0.85795456 - batches: 66\n",
            "Epoch 31/128 - 0.44s - loss: 62.453312 - acc: 0.86174244 - batches: 66\n",
            "Epoch 32/128 - 0.45s - loss: 62.343872 - acc: 0.86174244 - batches: 66\n",
            "Epoch 33/128 - 0.44s - loss: 62.24183 - acc: 0.86742425 - batches: 66\n",
            "Epoch 34/128 - 0.47s - loss: 62.146107 - acc: 0.86742425 - batches: 66\n",
            "Epoch 35/128 - 0.44s - loss: 62.0549 - acc: 0.8655303 - batches: 66\n",
            "Epoch 36/128 - 0.44s - loss: 61.96831 - acc: 0.86174244 - batches: 66\n",
            "Epoch 37/128 - 0.45s - loss: 61.886295 - acc: 0.86742425 - batches: 66\n",
            "Epoch 38/128 - 0.42s - loss: 61.80727 - acc: 0.8693182 - batches: 66\n",
            "Epoch 39/128 - 0.45s - loss: 61.729897 - acc: 0.87310606 - batches: 66\n",
            "Epoch 40/128 - 0.42s - loss: 61.6557 - acc: 0.87310606 - batches: 66\n",
            "Epoch 41/128 - 0.45s - loss: 61.586613 - acc: 0.875 - batches: 66\n",
            "Epoch 42/128 - 0.49s - loss: 61.520813 - acc: 0.875 - batches: 66\n",
            "Epoch 43/128 - 0.45s - loss: 61.458546 - acc: 0.875 - batches: 66\n",
            "Epoch 44/128 - 0.45s - loss: 61.399017 - acc: 0.875 - batches: 66\n",
            "Epoch 45/128 - 0.44s - loss: 61.341774 - acc: 0.875 - batches: 66\n",
            "Epoch 46/128 - 0.44s - loss: 61.28665 - acc: 0.875 - batches: 66\n",
            "Epoch 47/128 - 0.46s - loss: 61.233845 - acc: 0.875 - batches: 66\n",
            "Epoch 48/128 - 0.45s - loss: 61.183342 - acc: 0.87689394 - batches: 66\n",
            "Epoch 49/128 - 0.44s - loss: 61.134605 - acc: 0.8787879 - batches: 66\n",
            "Epoch 50/128 - 0.45s - loss: 61.087284 - acc: 0.8787879 - batches: 66\n",
            "Epoch 51/128 - 0.46s - loss: 61.041897 - acc: 0.8806818 - batches: 66\n",
            "Epoch 52/128 - 0.47s - loss: 60.998135 - acc: 0.8806818 - batches: 66\n",
            "Epoch 53/128 - 0.43s - loss: 60.955547 - acc: 0.8806818 - batches: 66\n",
            "Epoch 54/128 - 0.43s - loss: 60.914413 - acc: 0.8806818 - batches: 66\n",
            "Epoch 55/128 - 0.44s - loss: 60.874283 - acc: 0.88257575 - batches: 66\n",
            "Epoch 56/128 - 0.43s - loss: 60.8355 - acc: 0.88257575 - batches: 66\n",
            "Epoch 57/128 - 0.46s - loss: 60.797585 - acc: 0.88257575 - batches: 66\n",
            "Epoch 58/128 - 0.45s - loss: 60.76088 - acc: 0.88257575 - batches: 66\n",
            "Epoch 59/128 - 0.46s - loss: 60.72547 - acc: 0.88257575 - batches: 66\n",
            "Epoch 60/128 - 0.44s - loss: 60.690533 - acc: 0.8844697 - batches: 66\n",
            "Epoch 61/128 - 0.53s - loss: 60.6563 - acc: 0.8863636 - batches: 66\n",
            "Epoch 62/128 - 0.53s - loss: 60.62335 - acc: 0.8863636 - batches: 66\n",
            "Epoch 63/128 - 0.44s - loss: 60.59092 - acc: 0.8863636 - batches: 66\n",
            "Epoch 64/128 - 0.45s - loss: 60.559105 - acc: 0.8863636 - batches: 66\n",
            "Epoch 65/128 - 0.43s - loss: 60.528175 - acc: 0.8863636 - batches: 66\n",
            "Epoch 66/128 - 0.44s - loss: 60.49788 - acc: 0.8863636 - batches: 66\n",
            "Epoch 67/128 - 0.44s - loss: 60.468407 - acc: 0.8863636 - batches: 66\n",
            "Epoch 68/128 - 0.45s - loss: 60.439407 - acc: 0.8863636 - batches: 66\n",
            "Epoch 69/128 - 0.45s - loss: 60.410797 - acc: 0.8863636 - batches: 66\n",
            "Epoch 70/128 - 0.45s - loss: 60.383053 - acc: 0.8863636 - batches: 66\n",
            "Epoch 71/128 - 0.45s - loss: 60.355698 - acc: 0.8863636 - batches: 66\n",
            "Epoch 72/128 - 0.45s - loss: 60.329044 - acc: 0.8901515 - batches: 66\n",
            "Epoch 73/128 - 0.44s - loss: 60.302986 - acc: 0.89204544 - batches: 66\n",
            "Epoch 74/128 - 0.44s - loss: 60.277557 - acc: 0.8939394 - batches: 66\n",
            "Epoch 75/128 - 0.45s - loss: 60.252453 - acc: 0.8939394 - batches: 66\n",
            "Epoch 76/128 - 0.45s - loss: 60.22764 - acc: 0.8939394 - batches: 66\n",
            "Epoch 77/128 - 0.46s - loss: 60.203236 - acc: 0.8939394 - batches: 66\n",
            "Epoch 78/128 - 0.46s - loss: 60.179108 - acc: 0.8939394 - batches: 66\n",
            "Epoch 79/128 - 0.46s - loss: 60.155563 - acc: 0.8939394 - batches: 66\n",
            "Epoch 80/128 - 0.45s - loss: 60.132492 - acc: 0.8939394 - batches: 66\n",
            "Epoch 81/128 - 0.45s - loss: 60.109493 - acc: 0.8939394 - batches: 66\n",
            "Epoch 82/128 - 0.47s - loss: 60.08689 - acc: 0.8958333 - batches: 66\n",
            "Epoch 83/128 - 0.45s - loss: 60.064495 - acc: 0.8958333 - batches: 66\n",
            "Epoch 84/128 - 0.47s - loss: 60.04244 - acc: 0.8958333 - batches: 66\n",
            "Epoch 85/128 - 0.45s - loss: 60.020554 - acc: 0.8958333 - batches: 66\n",
            "Epoch 86/128 - 0.45s - loss: 59.999073 - acc: 0.89772725 - batches: 66\n",
            "Epoch 87/128 - 0.44s - loss: 59.9779 - acc: 0.89772725 - batches: 66\n",
            "Epoch 88/128 - 0.44s - loss: 59.956795 - acc: 0.8996212 - batches: 66\n",
            "Epoch 89/128 - 0.47s - loss: 59.936302 - acc: 0.8996212 - batches: 66\n",
            "Epoch 90/128 - 0.48s - loss: 59.916107 - acc: 0.8996212 - batches: 66\n",
            "Epoch 91/128 - 0.46s - loss: 59.896156 - acc: 0.8996212 - batches: 66\n",
            "Epoch 92/128 - 0.45s - loss: 59.87642 - acc: 0.8996212 - batches: 66\n",
            "Epoch 93/128 - 0.45s - loss: 59.85724 - acc: 0.8996212 - batches: 66\n",
            "Epoch 94/128 - 0.45s - loss: 59.83828 - acc: 0.8996212 - batches: 66\n",
            "Epoch 95/128 - 0.46s - loss: 59.81969 - acc: 0.8996212 - batches: 66\n",
            "Epoch 96/128 - 0.45s - loss: 59.801556 - acc: 0.8996212 - batches: 66\n",
            "Epoch 97/128 - 0.46s - loss: 59.783554 - acc: 0.8996212 - batches: 66\n",
            "Epoch 98/128 - 0.45s - loss: 59.765846 - acc: 0.8996212 - batches: 66\n",
            "Epoch 99/128 - 0.45s - loss: 59.74833 - acc: 0.9015151 - batches: 66\n",
            "Epoch 100/128 - 0.46s - loss: 59.73102 - acc: 0.90340906 - batches: 66\n",
            "Epoch 101/128 - 0.44s - loss: 59.713863 - acc: 0.90340906 - batches: 66\n",
            "Epoch 102/128 - 0.45s - loss: 59.697178 - acc: 0.90340906 - batches: 66\n",
            "Epoch 103/128 - 0.47s - loss: 59.680584 - acc: 0.90340906 - batches: 66\n",
            "Epoch 104/128 - 0.45s - loss: 59.664326 - acc: 0.90340906 - batches: 66\n",
            "Epoch 105/128 - 0.45s - loss: 59.64815 - acc: 0.90340906 - batches: 66\n",
            "Epoch 106/128 - 0.43s - loss: 59.632046 - acc: 0.90340906 - batches: 66\n",
            "Epoch 107/128 - 0.44s - loss: 59.616005 - acc: 0.90340906 - batches: 66\n",
            "Epoch 108/128 - 0.44s - loss: 59.600273 - acc: 0.90340906 - batches: 66\n",
            "Epoch 109/128 - 0.46s - loss: 59.584736 - acc: 0.90340906 - batches: 66\n",
            "Epoch 110/128 - 0.45s - loss: 59.569496 - acc: 0.90340906 - batches: 66\n",
            "Epoch 111/128 - 0.45s - loss: 59.55435 - acc: 0.90340906 - batches: 66\n",
            "Epoch 112/128 - 0.43s - loss: 59.539326 - acc: 0.905303 - batches: 66\n",
            "Epoch 113/128 - 0.44s - loss: 59.524536 - acc: 0.905303 - batches: 66\n",
            "Epoch 114/128 - 0.44s - loss: 59.510124 - acc: 0.905303 - batches: 66\n",
            "Epoch 115/128 - 0.44s - loss: 59.495728 - acc: 0.905303 - batches: 66\n",
            "Epoch 116/128 - 0.45s - loss: 59.481533 - acc: 0.905303 - batches: 66\n",
            "Epoch 117/128 - 0.46s - loss: 59.467484 - acc: 0.905303 - batches: 66\n",
            "Epoch 118/128 - 0.47s - loss: 59.45378 - acc: 0.905303 - batches: 66\n",
            "Epoch 119/128 - 0.44s - loss: 59.440086 - acc: 0.905303 - batches: 66\n",
            "Epoch 120/128 - 0.45s - loss: 59.426678 - acc: 0.905303 - batches: 66\n",
            "Epoch 121/128 - 0.44s - loss: 59.41344 - acc: 0.905303 - batches: 66\n",
            "Epoch 122/128 - 0.44s - loss: 59.400272 - acc: 0.905303 - batches: 66\n",
            "Epoch 123/128 - 0.46s - loss: 59.387325 - acc: 0.905303 - batches: 66\n",
            "Epoch 124/128 - 0.45s - loss: 59.374413 - acc: 0.905303 - batches: 66\n",
            "Epoch 125/128 - 0.46s - loss: 59.361732 - acc: 0.905303 - batches: 66\n",
            "Epoch 126/128 - 0.45s - loss: 59.349144 - acc: 0.905303 - batches: 66\n",
            "Epoch 127/128 - 0.46s - loss: 59.336735 - acc: 0.905303 - batches: 66\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "bd08e980-e125-4366-b275-61d42cb35e90",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bd08e980-e125-4366-b275-61d42cb35e90",
        "outputId": "259811cc-9f34-44f6-d10e-9c111e459edb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                  precision    recall  f1-score   support\n",
            "\n",
            "Gastroenterology       0.87      0.93      0.90        28\n",
            "       Neurology       0.88      0.88      0.88        24\n",
            "      Orthopedic       0.86      0.83      0.85        30\n",
            "         Urology       0.74      0.70      0.72        20\n",
            "\n",
            "        accuracy                           0.84       102\n",
            "       macro avg       0.84      0.83      0.83       102\n",
            "    weighted avg       0.84      0.84      0.84       102\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from sklearn.metrics import confusion_matrix, classification_report, accuracy_score\n",
        "\n",
        "preds = bert_sent_pipelineModel.transform(testData)\n",
        "\n",
        "preds_df = preds.select(\"category\",\"text\",\"class.result\").toPandas()\n",
        "\n",
        "preds_df[\"result\"] = preds_df[\"result\"].apply(lambda x : x[0])\n",
        "\n",
        "print (classification_report(preds_df[\"category\"], preds_df[\"result\"]))"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "20d94484-af06-4e57-a90e-86646224ac7c",
      "metadata": {
        "id": "20d94484-af06-4e57-a90e-86646224ac7c"
      },
      "source": [
        "## Bert Sentence Embeddings(sbiobert_base_cased_mli)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f4dc8bd6-dc76-4cdb-afd8-99ebf8959abb",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f4dc8bd6-dc76-4cdb-afd8-99ebf8959abb",
        "outputId": "5bddda1a-922f-44b3-b51a-dda3367a9d6e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "sbiobert_base_cased_mli download started this may take some time.\n",
            "Approximate size to download 384.3 MB\n",
            "[OK!]\n"
          ]
        }
      ],
      "source": [
        "document_assembler = DocumentAssembler()\\\n",
        "    .setInputCol(\"text\")\\\n",
        "    .setOutputCol(\"document\")\n",
        "\n",
        "sbert_embedder = BertSentenceEmbeddings.pretrained('sbiobert_base_cased_mli', 'en','clinical/models')\\\n",
        "    .setInputCols([\"document\"])\\\n",
        "    .setOutputCol(\"sentence_embeddings\")\\\n",
        "    .setMaxSentenceLength(512)\\\n",
        "    .setCaseSensitive(False)\n",
        "\n",
        "classsifier_dl = ClassifierDLApproach()\\\n",
        "    .setInputCols([\"sentence_embeddings\"])\\\n",
        "    .setOutputCol(\"class\")\\\n",
        "    .setLabelColumn(\"category\")\\\n",
        "    .setMaxEpochs(64)\\\n",
        "    .setBatchSize(8)\\\n",
        "    .setLr(0.0005)\\\n",
        "    .setDropout(0.3)\\\n",
        "    .setEnableOutputLogs(True)\n",
        "\n",
        "bert_sent_clf_pipeline = Pipeline(\n",
        "    stages = [\n",
        "    document_assembler,\n",
        "    sbert_embedder,\n",
        "    classsifier_dl\n",
        "])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "d3b10938-43ca-4bef-ad11-3ea0090f3707",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d3b10938-43ca-4bef-ad11-3ea0090f3707",
        "outputId": "10e99bcb-7dcf-42b2-daac-5370a9860a52"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 12.5 s, sys: 1.32 s, total: 13.8 s\n",
            "Wall time: 36min\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "bert_sent_pipelineModel = bert_sent_clf_pipeline.fit(trainingData)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!cd ~/annotator_logs/ && ls -lt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jC2FTNwlj0cT",
        "outputId": "a1ebd33e-f4bb-43ff-900a-03de61b97b86"
      },
      "id": "jC2FTNwlj0cT",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "total 8\n",
            "-rw-r--r-- 1 root root 4512 Feb 15 23:18 ClassifierDLApproach_40e43d4ec7d3.log\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!cat ~/annotator_logs/ClassifierDLApproach_40e43d4ec7d3.log"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LkESeUW2j0gp",
        "outputId": "916fbde3-1b1d-4129-fc96-98d127431e9c"
      },
      "id": "LkESeUW2j0gp",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training started - epochs: 64 - learning_rate: 5.0E-4 - batch_size: 8 - training_examples: 528 - classes: 4\n",
            "Epoch 0/64 - 1.08s - loss: 77.64776 - acc: 0.5814394 - batches: 66\n",
            "Epoch 1/64 - 0.86s - loss: 68.01735 - acc: 0.7234849 - batches: 66\n",
            "Epoch 2/64 - 0.86s - loss: 64.66168 - acc: 0.84090906 - batches: 66\n",
            "Epoch 3/64 - 0.84s - loss: 62.0315 - acc: 0.8806818 - batches: 66\n",
            "Epoch 4/64 - 0.85s - loss: 60.525566 - acc: 0.90909094 - batches: 66\n",
            "Epoch 5/64 - 0.86s - loss: 59.666126 - acc: 0.9185606 - batches: 66\n",
            "Epoch 6/64 - 0.84s - loss: 59.00746 - acc: 0.92424244 - batches: 66\n",
            "Epoch 7/64 - 0.83s - loss: 58.42534 - acc: 0.9261364 - batches: 66\n",
            "Epoch 8/64 - 0.82s - loss: 57.921482 - acc: 0.92992425 - batches: 66\n",
            "Epoch 9/64 - 0.85s - loss: 57.518993 - acc: 0.9337121 - batches: 66\n",
            "Epoch 10/64 - 0.83s - loss: 57.21453 - acc: 0.9318182 - batches: 66\n",
            "Epoch 11/64 - 0.82s - loss: 56.977867 - acc: 0.9337121 - batches: 66\n",
            "Epoch 12/64 - 0.82s - loss: 56.789948 - acc: 0.9337121 - batches: 66\n",
            "Epoch 13/64 - 0.80s - loss: 56.63347 - acc: 0.9337121 - batches: 66\n",
            "Epoch 14/64 - 0.83s - loss: 56.501022 - acc: 0.9337121 - batches: 66\n",
            "Epoch 15/64 - 0.80s - loss: 56.391026 - acc: 0.9337121 - batches: 66\n",
            "Epoch 16/64 - 0.81s - loss: 56.2921 - acc: 0.9337121 - batches: 66\n",
            "Epoch 17/64 - 0.81s - loss: 56.206467 - acc: 0.93560606 - batches: 66\n",
            "Epoch 18/64 - 0.82s - loss: 56.131165 - acc: 0.9375 - batches: 66\n",
            "Epoch 19/64 - 0.82s - loss: 56.063354 - acc: 0.93939394 - batches: 66\n",
            "Epoch 20/64 - 0.84s - loss: 56.003742 - acc: 0.93939394 - batches: 66\n",
            "Epoch 21/64 - 1.09s - loss: 55.94658 - acc: 0.9412879 - batches: 66\n",
            "Epoch 22/64 - 1.52s - loss: 55.895405 - acc: 0.9412879 - batches: 66\n",
            "Epoch 23/64 - 1.23s - loss: 55.848846 - acc: 0.9412879 - batches: 66\n",
            "Epoch 24/64 - 1.22s - loss: 55.80553 - acc: 0.9431818 - batches: 66\n",
            "Epoch 25/64 - 0.92s - loss: 55.76523 - acc: 0.9431818 - batches: 66\n",
            "Epoch 26/64 - 0.82s - loss: 55.726982 - acc: 0.94507575 - batches: 66\n",
            "Epoch 27/64 - 0.80s - loss: 55.691853 - acc: 0.94507575 - batches: 66\n",
            "Epoch 28/64 - 0.79s - loss: 55.658653 - acc: 0.94507575 - batches: 66\n",
            "Epoch 29/64 - 0.80s - loss: 55.626488 - acc: 0.94507575 - batches: 66\n",
            "Epoch 30/64 - 0.78s - loss: 55.59679 - acc: 0.9469697 - batches: 66\n",
            "Epoch 31/64 - 0.80s - loss: 55.5685 - acc: 0.9469697 - batches: 66\n",
            "Epoch 32/64 - 0.80s - loss: 55.541912 - acc: 0.9488636 - batches: 66\n",
            "Epoch 33/64 - 0.80s - loss: 55.51677 - acc: 0.9488636 - batches: 66\n",
            "Epoch 34/64 - 0.80s - loss: 55.492542 - acc: 0.9488636 - batches: 66\n",
            "Epoch 35/64 - 0.80s - loss: 55.469215 - acc: 0.9488636 - batches: 66\n",
            "Epoch 36/64 - 0.82s - loss: 55.447147 - acc: 0.9488636 - batches: 66\n",
            "Epoch 37/64 - 0.83s - loss: 55.425102 - acc: 0.9488636 - batches: 66\n",
            "Epoch 38/64 - 0.80s - loss: 55.404476 - acc: 0.9488636 - batches: 66\n",
            "Epoch 39/64 - 0.83s - loss: 55.384953 - acc: 0.9488636 - batches: 66\n",
            "Epoch 40/64 - 0.83s - loss: 55.36668 - acc: 0.95075756 - batches: 66\n",
            "Epoch 41/64 - 0.79s - loss: 55.347633 - acc: 0.95075756 - batches: 66\n",
            "Epoch 42/64 - 0.81s - loss: 55.330715 - acc: 0.95075756 - batches: 66\n",
            "Epoch 43/64 - 0.79s - loss: 55.313732 - acc: 0.95075756 - batches: 66\n",
            "Epoch 44/64 - 0.81s - loss: 55.297894 - acc: 0.95075756 - batches: 66\n",
            "Epoch 45/64 - 0.82s - loss: 55.281765 - acc: 0.95075756 - batches: 66\n",
            "Epoch 46/64 - 0.81s - loss: 55.267273 - acc: 0.95075756 - batches: 66\n",
            "Epoch 47/64 - 0.84s - loss: 55.252636 - acc: 0.95075756 - batches: 66\n",
            "Epoch 48/64 - 0.86s - loss: 55.238594 - acc: 0.95075756 - batches: 66\n",
            "Epoch 49/64 - 0.82s - loss: 55.225136 - acc: 0.9526515 - batches: 66\n",
            "Epoch 50/64 - 0.85s - loss: 55.211918 - acc: 0.9526515 - batches: 66\n",
            "Epoch 51/64 - 0.82s - loss: 55.198982 - acc: 0.9526515 - batches: 66\n",
            "Epoch 52/64 - 0.80s - loss: 55.1868 - acc: 0.9526515 - batches: 66\n",
            "Epoch 53/64 - 0.80s - loss: 55.17442 - acc: 0.9526515 - batches: 66\n",
            "Epoch 54/64 - 0.80s - loss: 55.16267 - acc: 0.9526515 - batches: 66\n",
            "Epoch 55/64 - 0.80s - loss: 55.150814 - acc: 0.9526515 - batches: 66\n",
            "Epoch 56/64 - 0.81s - loss: 55.13992 - acc: 0.9526515 - batches: 66\n",
            "Epoch 57/64 - 0.80s - loss: 55.128963 - acc: 0.95454544 - batches: 66\n",
            "Epoch 58/64 - 0.83s - loss: 55.11822 - acc: 0.95454544 - batches: 66\n",
            "Epoch 59/64 - 0.84s - loss: 55.107693 - acc: 0.95454544 - batches: 66\n",
            "Epoch 60/64 - 0.83s - loss: 55.097733 - acc: 0.95454544 - batches: 66\n",
            "Epoch 61/64 - 0.81s - loss: 55.087646 - acc: 0.95454544 - batches: 66\n",
            "Epoch 62/64 - 0.81s - loss: 55.07779 - acc: 0.95454544 - batches: 66\n",
            "Epoch 63/64 - 0.81s - loss: 55.068516 - acc: 0.95454544 - batches: 66\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "When we look at two Bert Sentence Embeddings, we see that **sbiobert_base_cased_mli** embeddings runs slower, but it has better results."
      ],
      "metadata": {
        "id": "RFb09BD_eERU"
      },
      "id": "RFb09BD_eERU"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "78f5ae53-2a50-4561-b1d2-ce5d300bd9d1",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "78f5ae53-2a50-4561-b1d2-ce5d300bd9d1",
        "outputId": "4a55c2a7-0c87-4a3c-c1f3-0c6428c3fae4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                  precision    recall  f1-score   support\n",
            "\n",
            "Gastroenterology       0.97      1.00      0.98        28\n",
            "       Neurology       0.88      0.96      0.92        24\n",
            "      Orthopedic       1.00      0.90      0.95        30\n",
            "         Urology       0.95      0.95      0.95        20\n",
            "\n",
            "        accuracy                           0.95       102\n",
            "       macro avg       0.95      0.95      0.95       102\n",
            "    weighted avg       0.95      0.95      0.95       102\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from sklearn.metrics import confusion_matrix, classification_report, accuracy_score\n",
        "\n",
        "preds = bert_sent_pipelineModel.transform(testData)\n",
        "\n",
        "preds_df = preds.select(\"category\",\"text\",\"class.result\").toPandas()\n",
        "\n",
        "preds_df[\"result\"] = preds_df[\"result\"].apply(lambda x : x[0])\n",
        "\n",
        "print (classification_report(preds_df[\"category\"], preds_df[\"result\"]))"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "2e7de4c1-8e95-46f5-9ed8-dfc7d0a7f6a1",
      "metadata": {
        "id": "2e7de4c1-8e95-46f5-9ed8-dfc7d0a7f6a1"
      },
      "source": [
        "## Clinical Longformer Embeddings\n",
        "\n",
        "This embeddings model was imported from Hugging Face( [link](https://huggingface.co/yikuan8/Clinical-Longformer)). Clinical-Longformer is a clinical knowledge enriched version of Longformer that was further pretrained using MIMIC-III clinical notes. It allows up to 4,096 tokens as the model input."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "da6cd83a-7c28-43e3-b1a6-cc2a08f32d25",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "da6cd83a-7c28-43e3-b1a6-cc2a08f32d25",
        "outputId": "42c98872-f947-44f7-bee2-9b9d3e5d4dda"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "clinical_longformer download started this may take some time.\n",
            "Approximate size to download 510.1 MB\n",
            "[OK!]\n"
          ]
        }
      ],
      "source": [
        "document_assembler = DocumentAssembler()\\\n",
        "    .setInputCol(\"text\")\\\n",
        "    .setOutputCol(\"document\")\n",
        "\n",
        "tokenizer = Tokenizer()\\\n",
        "    .setInputCols([\"document\"])\\\n",
        "    .setOutputCol(\"token\")\n",
        "\n",
        "lf_embeddings = LongformerEmbeddings.pretrained(\"clinical_longformer\", \"en\")\\\n",
        "  .setInputCols([\"document\", \"token\"])\\\n",
        "  .setOutputCol(\"embeddings\")\\\n",
        "  .setCaseSensitive(False)\\\n",
        "  .setMaxSentenceLength(512)\n",
        "\n",
        "embeddingsSentence = SentenceEmbeddings() \\\n",
        "    .setInputCols([\"document\", \"embeddings\"]) \\\n",
        "    .setOutputCol(\"sentence_embeddings\") \\\n",
        "    .setPoolingStrategy(\"AVERAGE\")\n",
        "\n",
        "classifierdl = ClassifierDLApproach()\\\n",
        "    .setInputCols([\"sentence_embeddings\"])\\\n",
        "    .setOutputCol(\"class\")\\\n",
        "    .setLabelColumn(\"category\")\\\n",
        "    .setBatchSize(8)\\\n",
        "    .setEnableOutputLogs(True)\\\n",
        "    .setMaxEpochs(64)\\\n",
        "    .setLr(0.003)\\\n",
        "    .setDropout(0.3)\n",
        "\n",
        "lf_pipeline = Pipeline(\n",
        "               stages = [\n",
        "                    document_assembler,\n",
        "                    tokenizer,\n",
        "                    lf_embeddings,\n",
        "                    embeddingsSentence,\n",
        "                    classifierdl\n",
        "               ])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e481551a-c72e-4f3e-8b48-f1342fe352ba",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e481551a-c72e-4f3e-8b48-f1342fe352ba",
        "outputId": "6632f83e-e4ae-41a2-8a17-d6055571e500"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 15.5 s, sys: 1.71 s, total: 17.2 s\n",
            "Wall time: 45min 7s\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "lf_model = lf_pipeline.fit(trainingData)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!cd ~/annotator_logs/ && ls -lt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Hn6zHMwkNzCL",
        "outputId": "e72291b2-6bc6-49f6-b59d-d6332e600dc0"
      },
      "id": "Hn6zHMwkNzCL",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "total 32\n",
            "-rw-r--r-- 1 root root 4507 Feb 15 21:53 ClassifierDLApproach_41c1c26d040d.log\n",
            "-rw-r--r-- 1 root root 9020 Feb 15 21:03 ClassifierDLApproach_726271d783f5.log\n",
            "-rw-r--r-- 1 root root 9075 Feb 15 20:52 ClassifierDLApproach_9571e7315f7f.log\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!cat ~/annotator_logs/ClassifierDLApproach_41c1c26d040d.log"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MyB5FnLAN0BK",
        "outputId": "a6d807ae-6e7c-4e47-fa39-0b03e7b64632"
      },
      "id": "MyB5FnLAN0BK",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training started - epochs: 64 - learning_rate: 0.003 - batch_size: 8 - training_examples: 528 - classes: 4\n",
            "Epoch 0/64 - 0.85s - loss: 90.93513 - acc: 0.33901516 - batches: 66\n",
            "Epoch 1/64 - 0.47s - loss: 79.88781 - acc: 0.51704544 - batches: 66\n",
            "Epoch 2/64 - 0.48s - loss: 69.131226 - acc: 0.6287879 - batches: 66\n",
            "Epoch 3/64 - 0.45s - loss: 67.21849 - acc: 0.7064394 - batches: 66\n",
            "Epoch 4/64 - 0.49s - loss: 67.160805 - acc: 0.72727275 - batches: 66\n",
            "Epoch 5/64 - 0.47s - loss: 65.39544 - acc: 0.7651515 - batches: 66\n",
            "Epoch 6/64 - 0.49s - loss: 64.52378 - acc: 0.7878788 - batches: 66\n",
            "Epoch 7/64 - 0.48s - loss: 63.951595 - acc: 0.79545456 - batches: 66\n",
            "Epoch 8/64 - 0.47s - loss: 63.488316 - acc: 0.8087121 - batches: 66\n",
            "Epoch 9/64 - 0.45s - loss: 63.128845 - acc: 0.81439394 - batches: 66\n",
            "Epoch 10/64 - 0.48s - loss: 62.890724 - acc: 0.8238636 - batches: 66\n",
            "Epoch 11/64 - 0.48s - loss: 62.742283 - acc: 0.8276515 - batches: 66\n",
            "Epoch 12/64 - 0.46s - loss: 62.6539 - acc: 0.8314394 - batches: 66\n",
            "Epoch 13/64 - 0.50s - loss: 62.596237 - acc: 0.844697 - batches: 66\n",
            "Epoch 14/64 - 0.46s - loss: 62.550564 - acc: 0.84659094 - batches: 66\n",
            "Epoch 15/64 - 0.48s - loss: 62.52191 - acc: 0.84659094 - batches: 66\n",
            "Epoch 16/64 - 0.46s - loss: 62.472702 - acc: 0.8503788 - batches: 66\n",
            "Epoch 17/64 - 0.49s - loss: 62.400314 - acc: 0.85227275 - batches: 66\n",
            "Epoch 18/64 - 0.46s - loss: 62.29359 - acc: 0.8503788 - batches: 66\n",
            "Epoch 19/64 - 0.48s - loss: 62.164814 - acc: 0.8541667 - batches: 66\n",
            "Epoch 20/64 - 0.45s - loss: 62.008617 - acc: 0.85227275 - batches: 66\n",
            "Epoch 21/64 - 0.49s - loss: 61.833057 - acc: 0.8560606 - batches: 66\n",
            "Epoch 22/64 - 0.46s - loss: 61.641197 - acc: 0.85795456 - batches: 66\n",
            "Epoch 23/64 - 0.48s - loss: 61.456573 - acc: 0.8636364 - batches: 66\n",
            "Epoch 24/64 - 0.46s - loss: 61.281345 - acc: 0.86742425 - batches: 66\n",
            "Epoch 25/64 - 0.46s - loss: 61.117706 - acc: 0.8693182 - batches: 66\n",
            "Epoch 26/64 - 0.50s - loss: 60.9597 - acc: 0.8693182 - batches: 66\n",
            "Epoch 27/64 - 0.49s - loss: 60.812515 - acc: 0.8712121 - batches: 66\n",
            "Epoch 28/64 - 0.48s - loss: 60.677513 - acc: 0.8712121 - batches: 66\n",
            "Epoch 29/64 - 0.47s - loss: 60.553432 - acc: 0.87310606 - batches: 66\n",
            "Epoch 30/64 - 0.49s - loss: 60.43208 - acc: 0.87310606 - batches: 66\n",
            "Epoch 31/64 - 0.47s - loss: 60.322094 - acc: 0.875 - batches: 66\n",
            "Epoch 32/64 - 0.49s - loss: 60.21695 - acc: 0.87689394 - batches: 66\n",
            "Epoch 33/64 - 0.47s - loss: 60.120228 - acc: 0.87689394 - batches: 66\n",
            "Epoch 34/64 - 0.48s - loss: 60.02935 - acc: 0.87689394 - batches: 66\n",
            "Epoch 35/64 - 0.50s - loss: 59.94398 - acc: 0.8787879 - batches: 66\n",
            "Epoch 36/64 - 0.62s - loss: 59.86377 - acc: 0.8806818 - batches: 66\n",
            "Epoch 37/64 - 0.49s - loss: 59.78877 - acc: 0.88257575 - batches: 66\n",
            "Epoch 38/64 - 0.50s - loss: 59.72009 - acc: 0.88257575 - batches: 66\n",
            "Epoch 39/64 - 0.47s - loss: 59.652424 - acc: 0.88257575 - batches: 66\n",
            "Epoch 40/64 - 0.48s - loss: 59.58981 - acc: 0.88257575 - batches: 66\n",
            "Epoch 41/64 - 0.47s - loss: 59.528316 - acc: 0.8863636 - batches: 66\n",
            "Epoch 42/64 - 0.49s - loss: 59.466576 - acc: 0.8863636 - batches: 66\n",
            "Epoch 43/64 - 0.46s - loss: 59.410206 - acc: 0.8863636 - batches: 66\n",
            "Epoch 44/64 - 0.46s - loss: 59.351063 - acc: 0.8901515 - batches: 66\n",
            "Epoch 45/64 - 0.48s - loss: 59.293743 - acc: 0.8901515 - batches: 66\n",
            "Epoch 46/64 - 0.48s - loss: 59.238415 - acc: 0.89204544 - batches: 66\n",
            "Epoch 47/64 - 0.47s - loss: 59.183674 - acc: 0.8939394 - batches: 66\n",
            "Epoch 48/64 - 0.47s - loss: 59.13104 - acc: 0.8939394 - batches: 66\n",
            "Epoch 49/64 - 0.46s - loss: 59.08186 - acc: 0.8939394 - batches: 66\n",
            "Epoch 50/64 - 0.47s - loss: 59.032764 - acc: 0.8939394 - batches: 66\n",
            "Epoch 51/64 - 0.49s - loss: 58.983345 - acc: 0.8958333 - batches: 66\n",
            "Epoch 52/64 - 0.48s - loss: 58.93694 - acc: 0.89772725 - batches: 66\n",
            "Epoch 53/64 - 0.47s - loss: 58.891224 - acc: 0.89772725 - batches: 66\n",
            "Epoch 54/64 - 0.48s - loss: 58.847046 - acc: 0.89772725 - batches: 66\n",
            "Epoch 55/64 - 0.47s - loss: 58.805775 - acc: 0.89772725 - batches: 66\n",
            "Epoch 56/64 - 0.48s - loss: 58.762978 - acc: 0.9015151 - batches: 66\n",
            "Epoch 57/64 - 0.47s - loss: 58.724373 - acc: 0.9015151 - batches: 66\n",
            "Epoch 58/64 - 0.49s - loss: 58.68602 - acc: 0.9015151 - batches: 66\n",
            "Epoch 59/64 - 0.46s - loss: 58.64993 - acc: 0.9015151 - batches: 66\n",
            "Epoch 60/64 - 0.48s - loss: 58.614594 - acc: 0.9015151 - batches: 66\n",
            "Epoch 61/64 - 0.48s - loss: 58.582016 - acc: 0.9015151 - batches: 66\n",
            "Epoch 62/64 - 0.47s - loss: 58.54909 - acc: 0.9015151 - batches: 66\n",
            "Epoch 63/64 - 0.48s - loss: 58.51908 - acc: 0.9015151 - batches: 66\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ad649a6a-950f-41e3-8bf8-f040acd26de8",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ad649a6a-950f-41e3-8bf8-f040acd26de8",
        "outputId": "9896d437-6b85-445e-fd20-43fef3e2f13d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                  precision    recall  f1-score   support\n",
            "\n",
            "Gastroenterology       0.77      0.96      0.86        28\n",
            "       Neurology       0.95      0.88      0.91        24\n",
            "      Orthopedic       0.91      0.97      0.94        30\n",
            "         Urology       0.92      0.60      0.73        20\n",
            "\n",
            "        accuracy                           0.87       102\n",
            "       macro avg       0.89      0.85      0.86       102\n",
            "    weighted avg       0.88      0.87      0.87       102\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from sklearn.metrics import confusion_matrix, classification_report, accuracy_score\n",
        "\n",
        "preds = lf_model.transform(testData)\n",
        "\n",
        "preds_df = preds.select(\"category\",\"text\",\"class.result\").toPandas()\n",
        "\n",
        "preds_df[\"result\"] = preds_df[\"result\"].apply(lambda x : x[0])\n",
        "\n",
        "print (classification_report(preds_df[\"category\"], preds_df[\"result\"]))"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Clinical Longformer Embeddings results are good, but takes a long time than Bert Sentence Embeddings & USE on Google Colab"
      ],
      "metadata": {
        "id": "-VVEg3OsfooV"
      },
      "id": "-VVEg3OsfooV"
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "bunyamin",
      "language": "python",
      "name": "bunyamin"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.12"
    },
    "colab": {
      "name": "3.Clinical_Longformer_vs_BertSentence_&_USE.ipynb",
      "provenance": [],
      "collapsed_sections": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}