PCL_Subtask_1_BERT.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ynPRbJviL-39"
      },
      "source": [
        "\n",
        "\n",
        "# Detecting PCL | Training Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z82c-Fcay0a3"
      },
      "source": [
        "### Importing the libraries"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "! pip install  transformers datasets"
      ],
      "metadata": {
        "id": "jx5l1_TqCdES"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PUNvqRAAJh0j"
      },
      "outputs": [],
      "source": [
        "import os.path\n",
        "import numpy as np\n",
        "import tensorflow as tf\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aoiYDiMs06GU"
      },
      "outputs": [],
      "source": [
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "364nmWXKy7Hu",
        "outputId": "0caae4f1-639d-4339-dbf7-a198b91a8d1c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fetching https://raw.githubusercontent.com/Perez-AlmendrosC/dontpatronizeme/master/semeval-2022/dont_patronize_me.py\n"
          ]
        }
      ],
      "source": [
        "from urllib import request\n",
        "\n",
        "module_url = f\"https://raw.githubusercontent.com/Perez-AlmendrosC/dontpatronizeme/master/semeval-2022/dont_patronize_me.py\"\n",
        "module_name = module_url.split('/')[-1]\n",
        "print(f'Fetching {module_url}')\n",
        "#with open(\"file_1.txt\") as f1, open(\"file_2.txt\") as f2\n",
        "with request.urlopen(module_url) as f, open(module_name,'w') as outf:\n",
        "  a = f.read()\n",
        "  outf.write(a.decode('utf-8'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wtlbKu5fy9BR"
      },
      "outputs": [],
      "source": [
        "from dont_patronize_me import DontPatronizeMe\n",
        "dpm = DontPatronizeMe('.', 'dontpatronizeme_pcl.tsv')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JEjlM2EazOf0"
      },
      "source": [
        "## Part 1: Data Preprocessing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ePywR8A4zaxT"
      },
      "source": [
        "### Loading the PCL dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Awazte7VzMiM"
      },
      "outputs": [],
      "source": [
        "dpm.load_task1()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "V9makkAWzNDj"
      },
      "outputs": [],
      "source": [
        "trids = pd.read_csv('train_semeval_parids-labels.csv')\n",
        "teids = pd.read_csv('dev_semeval_parids-labels.csv')\n",
        "\n",
        "trids.par_id = trids.par_id.astype(str)\n",
        "teids.par_id = teids.par_id.astype(str)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TvRw858jzPLa"
      },
      "outputs": [],
      "source": [
        "rows = [] # will contain par_id, label and text\n",
        "for idx in range(len(trids)):  \n",
        "  parid = trids.par_id[idx]\n",
        "  #print(parid)\n",
        "  # select row from original dataset to retrieve `text` and binary label\n",
        "  text = dpm.train_task1_df.loc[dpm.train_task1_df.par_id == parid].text.values[0]\n",
        "  label = dpm.train_task1_df.loc[dpm.train_task1_df.par_id == parid].label.values[0]\n",
        "  tag = dpm.train_task1_df.loc[dpm.train_task1_df.par_id == parid].label.values[0]\n",
        "  keyword = dpm.train_task1_df.loc[dpm.train_task1_df.par_id == parid].keyword.values[0]\n",
        "\n",
        "  rows.append({\n",
        "      'par_id':parid,\n",
        "      'text': text,\n",
        "      'keyword' : keyword,\n",
        "      'label':label\n",
        "  })\n",
        "  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MJBGgBcjzRvQ"
      },
      "outputs": [],
      "source": [
        "trdf1 = pd.DataFrame(rows)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HeoS64wZzThP"
      },
      "outputs": [],
      "source": [
        "rows = [] # will contain par_id, label and text\n",
        "for idx in range(len(teids)):  \n",
        "  parid = teids.par_id[idx]\n",
        "  #print(parid)\n",
        "  # select row from original dataset\n",
        "  text = dpm.train_task1_df.loc[dpm.train_task1_df.par_id == parid].text.values[0]\n",
        "  label = dpm.train_task1_df.loc[dpm.train_task1_df.par_id == parid].label.values[0]\n",
        "  keyword = dpm.train_task1_df.loc[dpm.train_task1_df.par_id == parid].keyword.values[0]\n",
        "\n",
        "  rows.append({\n",
        "      'par_id':parid,\n",
        "      'text': text,\n",
        "      'keyword' :keyword,\n",
        "      'label':label\n",
        "  })\n",
        "  "
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tedf1=pd.DataFrame(rows)"
      ],
      "metadata": {
        "id": "kGM4AIaJCrW4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class_weight = {0:2.80048445005996,1:11.8523601712888}\n",
        "# For try different weights for different values corresponding to 0 and 1\n",
        "# INS weight = 1/n\n",
        "# ISNS weight = 1/root(n)\n",
        "# ENS weight = (1-Beta)/(1-Beta^n)"
      ],
      "metadata": {
        "id": "nbNNm68vEpxO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Preparing the Dataset"
      ],
      "metadata": {
        "id": "p4LJumDvC15Z"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pIRBz3p_sz6k"
      },
      "outputs": [],
      "source": [
        "from datasets import Dataset\n",
        "train_raw = Dataset.from_pandas(trdf1)\n",
        "dev_raw = Dataset.from_pandas(tedf1)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(train_raw[0]['keyword'])"
      ],
      "metadata": {
        "id": "_vKjJdKpMRRs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')\n",
        "\n",
        "\n",
        "def tokenize_function(examples):\n",
        "  return tokenizer(text=examples['keyword'],text_pair=examples['text'],add_special_tokens=True, max_length=256, padding=\"max_length\", truncation=True)  \n",
        "train = train_raw.map(tokenize_function, batched=True)\n",
        "dev = dev_raw.map(tokenize_function , batched=True)"
      ],
      "metadata": {
        "id": "Tau8MaCY9Gx2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import DefaultDataCollator\n",
        "\n",
        "data_collator = DefaultDataCollator(return_tensors=\"tf\")"
      ],
      "metadata": {
        "id": "DCCmHSVH8c_W"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oHDQTH0uwTH_"
      },
      "outputs": [],
      "source": [
        "tf_train_dataset = train.to_tf_dataset(\n",
        "    columns=[\"attention_mask\", \"input_ids\", \"token_type_ids\"],\n",
        "    label_cols=[\"labels\"],\n",
        "    shuffle=True,\n",
        "    collate_fn=data_collator,\n",
        "    batch_size=16,\n",
        ")\n",
        "\n",
        "tf_validation_dataset = dev.to_tf_dataset(\n",
        "    columns=[\"attention_mask\", \"input_ids\", \"token_type_ids\"],\n",
        "    label_cols=[\"labels\"],\n",
        "    shuffle=False,\n",
        "    collate_fn=data_collator,\n",
        "    batch_size=16,\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Finetuning with Keras"
      ],
      "metadata": {
        "id": "nBi8e8PdC-vH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "from transformers import TFAutoModelForSequenceClassification\n",
        "\n",
        "model = TFAutoModelForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=1)"
      ],
      "metadata": {
        "id": "-37QpZi_8iMl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.compile(\n",
        "    optimizer=tf.keras.optimizers.Adam(learning_rate=2e-5),\n",
        "    loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n",
        "    metrics=tf.metrics.BinaryAccuracy(),\n",
        ")\n",
        "\n",
        "model.summary()\n",
        "\n",
        "model.fit(tf_train_dataset, validation_data=tf_validation_dataset, epochs=1, class_weight = class_weight)"
      ],
      "metadata": {
        "id": "vOxzWOOx8m1D"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "  PCL:Subtask 1 BERT",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}