testing-codes/Testing Transformer Models.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":8250654,"sourceType":"datasetVersion","datasetId":4895458}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Load the Dataset","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:24:00.621350Z","iopub.execute_input":"2024-05-10T19:24:00.622190Z","iopub.status.idle":"2024-05-10T19:24:01.006516Z","shell.execute_reply.started":"2024-05-10T19:24:00.622157Z","shell.execute_reply":"2024-05-10T19:24:01.005680Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"code","source":"ben_df_folder = \"/kaggle/input/modified-hate-speech-bengali-hindi/bengali-hate-speech/\"\nben_df_test = pd.read_csv(ben_df_folder + \"bengali-hate-speech_test.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:24:01.977536Z","iopub.execute_input":"2024-05-10T19:24:01.977986Z","iopub.status.idle":"2024-05-10T19:24:02.033635Z","shell.execute_reply.started":"2024-05-10T19:24:01.977959Z","shell.execute_reply":"2024-05-10T19:24:02.032709Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"hin_df_folder = \"/kaggle/input/modified-hate-speech-bengali-hindi/hindi-hate-speech/\"\nhin_df_test = pd.read_csv(hin_df_folder + \"hindi-hate-speech_test.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:24:02.465396Z","iopub.execute_input":"2024-05-10T19:24:02.465756Z","iopub.status.idle":"2024-05-10T19:24:02.501046Z","shell.execute_reply.started":"2024-05-10T19:24:02.465728Z","shell.execute_reply":"2024-05-10T19:24:02.500165Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"code","source":"ben_df_size = ben_df_test['text'].count()\nhin_df_size = hin_df_test['text'].count()","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:24:03.198044Z","iopub.execute_input":"2024-05-10T19:24:03.198445Z","iopub.status.idle":"2024-05-10T19:24:03.209751Z","shell.execute_reply.started":"2024-05-10T19:24:03.198414Z","shell.execute_reply":"2024-05-10T19:24:03.208631Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"markdown","source":"Find out the available devices","metadata":{}},{"cell_type":"code","source":"import torch","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:24:04.293931Z","iopub.execute_input":"2024-05-10T19:24:04.294690Z","iopub.status.idle":"2024-05-10T19:24:07.850748Z","shell.execute_reply.started":"2024-05-10T19:24:04.294657Z","shell.execute_reply":"2024-05-10T19:24:07.849861Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:24:07.852742Z","iopub.execute_input":"2024-05-10T19:24:07.853875Z","iopub.status.idle":"2024-05-10T19:24:07.879514Z","shell.execute_reply.started":"2024-05-10T19:24:07.853845Z","shell.execute_reply":"2024-05-10T19:24:07.878494Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"markdown","source":"# Load the Required Model","metadata":{}},{"cell_type":"code","source":"from transformers import pipeline","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:24:07.880864Z","iopub.execute_input":"2024-05-10T19:24:07.881635Z","iopub.status.idle":"2024-05-10T19:24:22.843185Z","shell.execute_reply.started":"2024-05-10T19:24:07.881579Z","shell.execute_reply":"2024-05-10T19:24:22.842308Z"},"trusted":true},"execution_count":7,"outputs":[{"name":"stderr","text":"2024-05-10 19:24:11.545806: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n2024-05-10 19:24:11.545916: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n2024-05-10 19:24:11.682999: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n","output_type":"stream"}]},{"cell_type":"code","source":"pipe = pipeline(\n    \"text-classification\",\n    model=\"kingshukroy/twhin-bert-base-hate-speech-ben-hin\",\n    device=device,\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:24:22.844951Z","iopub.execute_input":"2024-05-10T19:24:22.845627Z","iopub.status.idle":"2024-05-10T19:24:32.340739Z","shell.execute_reply.started":"2024-05-10T19:24:22.845589Z","shell.execute_reply":"2024-05-10T19:24:32.339822Z"},"trusted":true},"execution_count":8,"outputs":[{"output_type":"display_data","data":{"text/plain":"config.json:   0%|          | 0.00/934 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5489511926644f07a4c07fda068d12d2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"model.safetensors:   0%|          | 0.00/1.12G [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0dbcc81c537542c49907cf6ab3cd6b02"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"tokenizer_config.json:   0%|          | 0.00/1.18k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"355dd9ca589848a99424bae3b2a54cf3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"tokenizer.json:   0%|          | 0.00/17.1M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4813b7f23bee47cba07e31b216efbff5"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"special_tokens_map.json:   0%|          | 0.00/964 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f24e856236bf4b389ca08e98263a5e29"}},"metadata":{}}]},{"cell_type":"markdown","source":"## Get the features","metadata":{}},{"cell_type":"code","source":"features = list(pipe.model.config.label2id.keys())\nnum_features = len(features)\nfeatures, num_features","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:24:32.342469Z","iopub.execute_input":"2024-05-10T19:24:32.342808Z","iopub.status.idle":"2024-05-10T19:24:32.350969Z","shell.execute_reply.started":"2024-05-10T19:24:32.342779Z","shell.execute_reply":"2024-05-10T19:24:32.349808Z"},"trusted":true},"execution_count":9,"outputs":[{"execution_count":9,"output_type":"execute_result","data":{"text/plain":"(['defamation', 'hate', 'non-hate', 'violence', 'vulgar'], 5)"},"metadata":{}}]},{"cell_type":"markdown","source":"# Create a Labels Column in the DataFrane","metadata":{}},{"cell_type":"code","source":"ben_df_test['labels'] = ben_df_test.drop(['text'], axis=1).apply(lambda row: [row[x] for x in features], axis=1)\nhin_df_test['labels'] = hin_df_test.drop(['text'], axis=1).apply(lambda row: [row[x] for x in features], axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:24:32.352315Z","iopub.execute_input":"2024-05-10T19:24:32.352686Z","iopub.status.idle":"2024-05-10T19:24:36.808320Z","shell.execute_reply.started":"2024-05-10T19:24:32.352660Z","shell.execute_reply":"2024-05-10T19:24:36.807440Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"markdown","source":"# Merge the Two DataFranes","metadata":{}},{"cell_type":"code","source":"merged_df_test = pd.concat([ben_df_test, hin_df_test], axis=0, ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:24:36.811031Z","iopub.execute_input":"2024-05-10T19:24:36.811340Z","iopub.status.idle":"2024-05-10T19:24:36.817339Z","shell.execute_reply.started":"2024-05-10T19:24:36.811308Z","shell.execute_reply":"2024-05-10T19:24:36.816321Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"markdown","source":"# Convert the DataFrames into PyTorch Datasets","metadata":{}},{"cell_type":"code","source":"from torch.utils.data import Dataset","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:24:36.818810Z","iopub.execute_input":"2024-05-10T19:24:36.819247Z","iopub.status.idle":"2024-05-10T19:24:36.827481Z","shell.execute_reply.started":"2024-05-10T19:24:36.819213Z","shell.execute_reply":"2024-05-10T19:24:36.826564Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"code","source":"class TextDatasetWrapper(Dataset):\n    def __init__(self, df, text_col='text'):\n        self.df = df\n        self.__text_col__ = text_col\n    def __getitem__(self, index):\n        text = self.df[self.__text_col__][index]\n        return text\n    def __len__(self):\n        return len(self.df)\n    \nben_dataset_text = TextDatasetWrapper(ben_df_test)\nhin_dataset_text = TextDatasetWrapper(hin_df_test)\nmerged_dataset_text = TextDatasetWrapper(merged_df_test)","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:24:36.828568Z","iopub.execute_input":"2024-05-10T19:24:36.828829Z","iopub.status.idle":"2024-05-10T19:24:36.838034Z","shell.execute_reply.started":"2024-05-10T19:24:36.828807Z","shell.execute_reply":"2024-05-10T19:24:36.837190Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"markdown","source":"# Get the predicted Labels","metadata":{}},{"cell_type":"code","source":"tokenize_args = {\n    \"padding\": \"max_length\",\n    \"max_length\": 256,\n    \"truncation\": True\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:24:36.839249Z","iopub.execute_input":"2024-05-10T19:24:36.839616Z","iopub.status.idle":"2024-05-10T19:24:36.849110Z","shell.execute_reply.started":"2024-05-10T19:24:36.839565Z","shell.execute_reply":"2024-05-10T19:24:36.848235Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"from tqdm.auto import tqdm","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:24:36.850265Z","iopub.execute_input":"2024-05-10T19:24:36.851181Z","iopub.status.idle":"2024-05-10T19:24:36.859165Z","shell.execute_reply.started":"2024-05-10T19:24:36.851131Z","shell.execute_reply":"2024-05-10T19:24:36.858343Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"code","source":"def get_labels_from_scores(scores_list, threshold = 0.5):\n    scores_df = pd.DataFrame(scores_list)\n    scores_df = scores_df.T.reset_index(drop=True)\n    scores_df.columns = scores_df.iloc[0]\n    scores_df = scores_df[1:].reset_index(drop=True)\n    probs_list = [scores_df[x].values[0] for x in features]\n    labels_list = [1 if x > threshold else 0 for x in probs_list]\n    return labels_list","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:24:36.860124Z","iopub.execute_input":"2024-05-10T19:24:36.860462Z","iopub.status.idle":"2024-05-10T19:24:36.870773Z","shell.execute_reply.started":"2024-05-10T19:24:36.860436Z","shell.execute_reply":"2024-05-10T19:24:36.869871Z"},"trusted":true},"execution_count":16,"outputs":[]},{"cell_type":"code","source":"merged_pred_labels = []\nfor pred in tqdm(pipe(merged_dataset_text, batch_size=32, top_k=None, **tokenize_args), total=len(merged_dataset_text)):\n    merged_pred_labels.append(get_labels_from_scores(pred))\nben_pred_labels = merged_pred_labels[:ben_df_size]\nhin_pred_labels = merged_pred_labels[ben_df_size:]","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:24:36.873433Z","iopub.execute_input":"2024-05-10T19:24:36.874367Z","iopub.status.idle":"2024-05-10T19:25:27.157113Z","shell.execute_reply.started":"2024-05-10T19:24:36.874332Z","shell.execute_reply":"2024-05-10T19:25:27.155830Z"},"trusted":true},"execution_count":17,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4868 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3ef067a2cb0d4c2e912af1c95f1c5691"}},"metadata":{}}]},{"cell_type":"markdown","source":"# Performence Matrices for the Predictions","metadata":{}},{"cell_type":"code","source":"import sklearn.metrics as matrices","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:25:27.159014Z","iopub.execute_input":"2024-05-10T19:25:27.159794Z","iopub.status.idle":"2024-05-10T19:25:27.165026Z","shell.execute_reply.started":"2024-05-10T19:25:27.159751Z","shell.execute_reply":"2024-05-10T19:25:27.163878Z"},"trusted":true},"execution_count":18,"outputs":[]},{"cell_type":"code","source":"def show_report(actual_labels, y_pred):\n    zero_division_value = 0\n    print('Accuracy Score: ', matrices.accuracy_score(actual_labels, y_pred))\n    print('F1 Score: ', matrices.f1_score(actual_labels, y_pred, average='micro',\n                                          zero_division=zero_division_value))\n    print('ROC AUC: ', matrices.roc_auc_score(actual_labels, y_pred, average='micro'))\n    print('Hamming Loss: ', matrices.hamming_loss(actual_labels, y_pred))\n    print('Jaccard Score: ', matrices.jaccard_score(actual_labels, y_pred, average='micro'))\n    print('Zero-One Loss:', matrices.zero_one_loss(actual_labels, y_pred))\n\n    print('\\nClassification Report :-\\n',\n          matrices.classification_report(\n              actual_labels,\n              y_pred,\n              output_dict=False,\n              target_names=features,\n              zero_division=zero_division_value,\n          )\n         )","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:25:27.166406Z","iopub.execute_input":"2024-05-10T19:25:27.166700Z","iopub.status.idle":"2024-05-10T19:25:27.179586Z","shell.execute_reply.started":"2024-05-10T19:25:27.166677Z","shell.execute_reply":"2024-05-10T19:25:27.178626Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"markdown","source":"## Performance Matrices on Bengali Dataset","metadata":{}},{"cell_type":"code","source":"show_report(list(ben_df_test['labels']), ben_pred_labels)","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:25:27.180763Z","iopub.execute_input":"2024-05-10T19:25:27.181066Z","iopub.status.idle":"2024-05-10T19:25:27.347649Z","shell.execute_reply.started":"2024-05-10T19:25:27.181031Z","shell.execute_reply":"2024-05-10T19:25:27.346613Z"},"trusted":true},"execution_count":20,"outputs":[{"name":"stdout","text":"Accuracy Score:  0.7471228615863141\nF1 Score:  0.8188891870136837\nROC AUC:  0.881683922226718\nHamming Loss:  0.08398133748055987\nJaccard Score:  0.6933212176283507\nZero-One Loss: 0.25287713841368586\n\nClassification Report :-\n               precision    recall  f1-score   support\n\n  defamation       0.69      0.76      0.72       828\n        hate       0.64      0.58      0.61       364\n    non-hate       0.95      0.93      0.94      1548\n    violence       0.75      0.76      0.76       482\n      vulgar       0.87      0.80      0.83       511\n\n   micro avg       0.82      0.82      0.82      3733\n   macro avg       0.78      0.77      0.77      3733\nweighted avg       0.82      0.82      0.82      3733\n samples avg       0.82      0.83      0.82      3733\n\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## Performance Matrices on Hinidi Dataset","metadata":{}},{"cell_type":"code","source":"show_report(list(hin_df_test['labels']), hin_pred_labels)","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:25:27.349158Z","iopub.execute_input":"2024-05-10T19:25:27.349885Z","iopub.status.idle":"2024-05-10T19:25:27.433751Z","shell.execute_reply.started":"2024-05-10T19:25:27.349847Z","shell.execute_reply":"2024-05-10T19:25:27.432831Z"},"trusted":true},"execution_count":21,"outputs":[{"name":"stdout","text":"Accuracy Score:  0.7513611615245009\nF1 Score:  0.7864010989010989\nROC AUC:  0.8628545175202422\nHamming Loss:  0.07525710828796128\nJaccard Score:  0.6479909451046972\nZero-One Loss: 0.24863883847549906\n\nClassification Report :-\n               precision    recall  f1-score   support\n\n  defamation       0.47      0.28      0.35       169\n        hate       0.53      0.50      0.51       234\n    non-hate       0.98      0.97      0.97       873\n    violence       0.00      0.00      0.00         0\n      vulgar       0.59      0.61      0.60       219\n\n   micro avg       0.81      0.77      0.79      1495\n   macro avg       0.51      0.47      0.49      1495\nweighted avg       0.79      0.77      0.78      1495\n samples avg       0.66      0.66      0.65      1495\n\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## Performance Matrices on Merged Dataset\nMerged Dataset = Bengali Dataset + Hindi Dataset","metadata":{}},{"cell_type":"code","source":"show_report(list(merged_df_test['labels']), merged_pred_labels)","metadata":{"execution":{"iopub.status.busy":"2024-05-10T19:25:27.435970Z","iopub.execute_input":"2024-05-10T19:25:27.436311Z","iopub.status.idle":"2024-05-10T19:25:27.625917Z","shell.execute_reply.started":"2024-05-10T19:25:27.436269Z","shell.execute_reply":"2024-05-10T19:25:27.624874Z"},"trusted":true},"execution_count":22,"outputs":[{"name":"stdout","text":"Accuracy Score:  0.7485620377978636\nF1 Score:  0.8097626857032606\nROC AUC:  0.876778286446135\nHamming Loss:  0.0810188989317995\nJaccard Score:  0.6803371697195656\nZero-One Loss: 0.25143796220213643\n\nClassification Report :-\n               precision    recall  f1-score   support\n\n  defamation       0.66      0.68      0.67       997\n        hate       0.60      0.55      0.57       598\n    non-hate       0.96      0.94      0.95      2421\n    violence       0.75      0.76      0.76       482\n      vulgar       0.78      0.74      0.76       730\n\n   micro avg       0.82      0.80      0.81      5228\n   macro avg       0.75      0.74      0.74      5228\nweighted avg       0.82      0.80      0.81      5228\n samples avg       0.77      0.77      0.76      5228\n\n","output_type":"stream"}]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}