bert-twitter-sentiment-classifier.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# BERT - Twitter Sentiment Classifier","metadata":{}},{"cell_type":"code","source":"\nimport torch\nfrom torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\nimport torch.nn.functional as F\nfrom transformers import BertTokenizer, BertConfig,AdamW, BertForSequenceClassification,get_linear_schedule_with_warmup\n\n\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix,classification_report\n# Import and evaluate each test batch using Matthew's correlation coefficient\nfrom sklearn.metrics import accuracy_score,matthews_corrcoef\n\nfrom tqdm import tqdm, trange,tnrange,tqdm_notebook\nimport random\nimport os\nimport io\n% matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2021-06-07T14:02:59.149195Z","iopub.execute_input":"2021-06-07T14:02:59.149515Z","iopub.status.idle":"2021-06-07T14:02:59.160990Z","shell.execute_reply.started":"2021-06-07T14:02:59.149484Z","shell.execute_reply":"2021-06-07T14:02:59.159168Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stderr","text":"UsageError: Line magic function `%` not found.\n","output_type":"stream"}]},{"cell_type":"code","source":"# identify and specify the GPU as the device, later in training loop we will load data into device\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nn_gpu = torch.cuda.device_count()\ntorch.cuda.get_device_name(0)\n\nSEED = 19\n\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\nif device == torch.device(\"cuda\"):\n    torch.cuda.manual_seed_all(SEED)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T14:03:05.636963Z","iopub.execute_input":"2021-06-07T14:03:05.637281Z","iopub.status.idle":"2021-06-07T14:03:05.687948Z","shell.execute_reply.started":"2021-06-07T14:03:05.637252Z","shell.execute_reply":"2021-06-07T14:03:05.687054Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/twitter-sentiment-dataset/Twitter_Data.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-06-07T14:03:08.348651Z","iopub.execute_input":"2021-06-07T14:03:08.348984Z","iopub.status.idle":"2021-06-07T14:03:08.913459Z","shell.execute_reply.started":"2021-06-07T14:03:08.348956Z","shell.execute_reply":"2021-06-07T14:03:08.912647Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"df_train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-06-07T14:03:12.659845Z","iopub.execute_input":"2021-06-07T14:03:12.660176Z","iopub.status.idle":"2021-06-07T14:03:12.690073Z","shell.execute_reply.started":"2021-06-07T14:03:12.660145Z","shell.execute_reply":"2021-06-07T14:03:12.689122Z"},"trusted":true},"execution_count":6,"outputs":[{"execution_count":6,"output_type":"execute_result","data":{"text/plain":"clean_text    4\ncategory      7\ndtype: int64"},"metadata":{}}]},{"cell_type":"markdown","source":"#### Observation - Requires data cleaning","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-07T14:03:15.435413Z","iopub.execute_input":"2021-06-07T14:03:15.435822Z","iopub.status.idle":"2021-06-07T14:03:15.454209Z","shell.execute_reply.started":"2021-06-07T14:03:15.435788Z","shell.execute_reply":"2021-06-07T14:03:15.453293Z"},"trusted":true},"execution_count":7,"outputs":[{"execution_count":7,"output_type":"execute_result","data":{"text/plain":"                                          clean_text  category\n0  when modi promised “minimum government maximum...      -1.0\n1  talk all the nonsense and continue all the dra...       0.0\n2  what did just say vote for modi  welcome bjp t...       1.0\n3  asking his supporters prefix chowkidar their n...       1.0\n4  answer who among these the most powerful world...       1.0","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>clean_text</th>\n      <th>category</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>when modi promised “minimum government maximum...</td>\n      <td>-1.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>talk all the nonsense and continue all the dra...</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>what did just say vote for modi  welcome bjp t...</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>asking his supporters prefix chowkidar their n...</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>answer who among these the most powerful world...</td>\n      <td>1.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"markdown","source":"## Target Distribution","metadata":{}},{"cell_type":"code","source":"df_train['category'].unique()","metadata":{"execution":{"iopub.status.busy":"2021-06-07T14:03:19.107848Z","iopub.execute_input":"2021-06-07T14:03:19.108171Z","iopub.status.idle":"2021-06-07T14:03:19.120126Z","shell.execute_reply.started":"2021-06-07T14:03:19.108143Z","shell.execute_reply":"2021-06-07T14:03:19.118885Z"},"trusted":true},"execution_count":8,"outputs":[{"execution_count":8,"output_type":"execute_result","data":{"text/plain":"array([-1.,  0.,  1., nan])"},"metadata":{}}]},{"cell_type":"code","source":"df_train['category'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-06-07T14:03:25.749153Z","iopub.execute_input":"2021-06-07T14:03:25.749487Z","iopub.status.idle":"2021-06-07T14:03:25.761059Z","shell.execute_reply.started":"2021-06-07T14:03:25.749455Z","shell.execute_reply":"2021-06-07T14:03:25.759776Z"},"trusted":true},"execution_count":9,"outputs":[{"execution_count":9,"output_type":"execute_result","data":{"text/plain":" 1.0    72250\n 0.0    55213\n-1.0    35510\nName: category, dtype: int64"},"metadata":{}}]},{"cell_type":"markdown","source":"# Data cleaning","metadata":{}},{"cell_type":"markdown","source":"### Ignoring the null values","metadata":{}},{"cell_type":"code","source":"df_train = df_train[~df_train['category'].isnull()]","metadata":{"execution":{"iopub.status.busy":"2021-06-07T14:03:28.283511Z","iopub.execute_input":"2021-06-07T14:03:28.283832Z","iopub.status.idle":"2021-06-07T14:03:28.300712Z","shell.execute_reply.started":"2021-06-07T14:03:28.283806Z","shell.execute_reply":"2021-06-07T14:03:28.299907Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"df_train = df_train[~df_train['clean_text'].isnull()]","metadata":{"execution":{"iopub.status.busy":"2021-06-07T14:03:30.356626Z","iopub.execute_input":"2021-06-07T14:03:30.356951Z","iopub.status.idle":"2021-06-07T14:03:30.389509Z","shell.execute_reply.started":"2021-06-07T14:03:30.356923Z","shell.execute_reply":"2021-06-07T14:03:30.388504Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"markdown","source":"# Target Encodeing","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nlabelencoder = LabelEncoder()\ndf_train['category_1'] = labelencoder.fit_transform(df_train['category'])","metadata":{"execution":{"iopub.status.busy":"2021-06-07T14:03:35.014252Z","iopub.execute_input":"2021-06-07T14:03:35.014597Z","iopub.status.idle":"2021-06-07T14:03:35.030171Z","shell.execute_reply.started":"2021-06-07T14:03:35.014559Z","shell.execute_reply":"2021-06-07T14:03:35.029315Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"code","source":"df_train[['category','category_1']].drop_duplicates(keep='first')","metadata":{"execution":{"iopub.status.busy":"2021-06-07T14:03:37.412047Z","iopub.execute_input":"2021-06-07T14:03:37.412377Z","iopub.status.idle":"2021-06-07T14:03:37.433360Z","shell.execute_reply.started":"2021-06-07T14:03:37.412347Z","shell.execute_reply":"2021-06-07T14:03:37.432350Z"},"trusted":true},"execution_count":13,"outputs":[{"execution_count":13,"output_type":"execute_result","data":{"text/plain":"   category  category_1\n0      -1.0           0\n1       0.0           1\n2       1.0           2","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>category</th>\n      <th>category_1</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>-1.0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1.0</td>\n      <td>2</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"df_train.rename(columns={'category_1':'label'},inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T14:03:39.531877Z","iopub.execute_input":"2021-06-07T14:03:39.532192Z","iopub.status.idle":"2021-06-07T14:03:39.536372Z","shell.execute_reply.started":"2021-06-07T14:03:39.532162Z","shell.execute_reply":"2021-06-07T14:03:39.535520Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"markdown","source":"# Data Preperation for BERT model","metadata":{}},{"cell_type":"code","source":"## create label and sentence list\nsentences = df_train.clean_text.values\n\n#check distribution of data based on labels\nprint(\"Distribution of data based on labels: \",df_train.label.value_counts())\n\n# Set the maximum sequence length. The longest sequence in our training set is 47, but we'll leave room on the end anyway. \n# In the original paper, the authors used a length of 512.\nMAX_LEN = 256\n\n## Import BERT tokenizer, that is used to convert our text into tokens that corresponds to BERT library\ntokenizer = BertTokenizer.from_pretrained('bert-base-uncased',do_lower_case=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T14:03:42.931757Z","iopub.execute_input":"2021-06-07T14:03:42.932060Z","iopub.status.idle":"2021-06-07T14:03:46.933297Z","shell.execute_reply.started":"2021-06-07T14:03:42.932034Z","shell.execute_reply":"2021-06-07T14:03:46.932496Z"},"trusted":true},"execution_count":15,"outputs":[{"name":"stdout","text":"Distribution of data based on labels:  2    72249\n1    55211\n0    35509\nName: label, dtype: int64\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/232k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e986fbfa0c5e4d70827124ce80bd6f52"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/28.0 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"611fcc0e726141bbb66284ac5f2de254"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/466k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1d45e9f95dfc48d38eb980fc360fadb5"}},"metadata":{}}]},{"cell_type":"code","source":"input_ids = [tokenizer.encode(sent, add_special_tokens=True,max_length=MAX_LEN,pad_to_max_length=True,truncation=True) for sent in sentences]","metadata":{"execution":{"iopub.status.busy":"2021-06-07T14:03:50.876689Z","iopub.execute_input":"2021-06-07T14:03:50.877017Z","iopub.status.idle":"2021-06-07T14:05:59.606024Z","shell.execute_reply.started":"2021-06-07T14:03:50.876986Z","shell.execute_reply":"2021-06-07T14:05:59.605212Z"},"trusted":true},"execution_count":16,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.7/site-packages/transformers/tokenization_utils_base.py:2079: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n  FutureWarning,\n","output_type":"stream"}]},{"cell_type":"code","source":"\nlabels = df_train.label.values\n\nprint(\"Actual sentence before tokenization: \",sentences[2])\nprint(\"Encoded Input from dataset: \",input_ids[2])\n\n## Create attention mask\nattention_masks = []\n## Create a mask of 1 for all input tokens and 0 for all padding tokens\nattention_masks = [[float(i>0) for i in seq] for seq in input_ids]\nprint(attention_masks[2])","metadata":{"execution":{"iopub.status.busy":"2021-06-07T14:06:10.808397Z","iopub.execute_input":"2021-06-07T14:06:10.808735Z","iopub.status.idle":"2021-06-07T14:06:18.247460Z","shell.execute_reply.started":"2021-06-07T14:06:10.808704Z","shell.execute_reply":"2021-06-07T14:06:18.246535Z"},"trusted":true},"execution_count":17,"outputs":[{"name":"stdout","text":"Actual sentence before tokenization:  what did just say vote for modi  welcome bjp told you rahul the main campaigner for modi think modi should just relax\nEncoded Input from dataset:  [101, 2054, 2106, 2074, 2360, 3789, 2005, 16913, 2072, 6160, 24954, 2409, 2017, 10958, 21886, 1996, 2364, 3049, 2121, 2005, 16913, 2072, 2228, 16913, 2072, 2323, 2074, 9483, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\n","output_type":"stream"}]},{"cell_type":"code","source":"train_inputs,validation_inputs,train_labels,validation_labels = train_test_split(input_ids,labels,random_state=41,test_size=0.1)\ntrain_masks,validation_masks,_,_ = train_test_split(attention_masks,input_ids,random_state=41,test_size=0.1)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T14:06:23.566366Z","iopub.execute_input":"2021-06-07T14:06:23.566707Z","iopub.status.idle":"2021-06-07T14:06:23.750370Z","shell.execute_reply.started":"2021-06-07T14:06:23.566672Z","shell.execute_reply":"2021-06-07T14:06:23.749539Z"},"trusted":true},"execution_count":18,"outputs":[]},{"cell_type":"code","source":"# convert all our data into torch tensors, required data type for our model\ntrain_inputs = torch.tensor(train_inputs)\nvalidation_inputs = torch.tensor(validation_inputs)\ntrain_labels = torch.tensor(train_labels)\nvalidation_labels = torch.tensor(validation_labels)\ntrain_masks = torch.tensor(train_masks)\nvalidation_masks = torch.tensor(validation_masks)\n\n# Select a batch size for training. For fine-tuning BERT on a specific task, the authors recommend a batch size of 16 or 32\nbatch_size = 32\n\n# Create an iterator of our data with torch DataLoader. This helps save on memory during training because, unlike a for loop, \n# with an iterator the entire dataset does not need to be loaded into memory\ntrain_data = TensorDataset(train_inputs,train_masks,train_labels)\ntrain_sampler = RandomSampler(train_data)\ntrain_dataloader = DataLoader(train_data,sampler=train_sampler,batch_size=batch_size)\n\nvalidation_data = TensorDataset(validation_inputs,validation_masks,validation_labels)\nvalidation_sampler = RandomSampler(validation_data)\nvalidation_dataloader = DataLoader(validation_data,sampler=validation_sampler,batch_size=batch_size)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T14:06:26.423004Z","iopub.execute_input":"2021-06-07T14:06:26.423326Z","iopub.status.idle":"2021-06-07T14:06:33.899862Z","shell.execute_reply.started":"2021-06-07T14:06:26.423295Z","shell.execute_reply":"2021-06-07T14:06:33.898988Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"markdown","source":"### Lets see how the training data looks like","metadata":{}},{"cell_type":"code","source":"train_data[0]","metadata":{"execution":{"iopub.status.busy":"2021-06-07T14:07:12.376034Z","iopub.execute_input":"2021-06-07T14:07:12.376492Z","iopub.status.idle":"2021-06-07T14:07:12.441217Z","shell.execute_reply.started":"2021-06-07T14:07:12.376447Z","shell.execute_reply":"2021-06-07T14:07:12.440181Z"},"trusted":true},"execution_count":20,"outputs":[{"execution_count":20,"output_type":"execute_result","data":{"text/plain":"(tensor([  101, 16913,  2072,  4067,  2003,  3217,  6529,  2003,  3217,  9058,\n          6950,  1998,  7558, 21665,  6505,  2634,  3352,  2686,  3565, 11452,\n          3519,  4283, 23556,  2008,  2634,  2085,  2686,  3565,  2373,  4489,\n           100,   102,     0,     0,     0,     0,     0,     0,     0,     0,\n             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n             0,     0,     0,     0,     0,     0]),\n tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n         1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0.,\n         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n         0., 0., 0., 0.]),\n tensor(2))"},"metadata":{}}]},{"cell_type":"markdown","source":"## Load BertForSequenceClassification, the pretrained BERT model with a single linear classification layer on top. ","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load BertForSequenceClassification, the pretrained BERT model with a single linear classification layer on top. \nmodel = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\", num_labels=3).to(device)\n\n# Parameters:\nlr = 2e-5\nadam_epsilon = 1e-8\n\n# Number of training epochs (authors recommend between 2 and 4)\nepochs = 3\n\nnum_warmup_steps = 0\nnum_training_steps = len(train_dataloader)*epochs\n\n### In Transformers, optimizer and schedules are splitted and instantiated like this:\noptimizer = AdamW(model.parameters(), lr=lr,eps=adam_epsilon,correct_bias=False)  # To reproduce BertAdam specific behavior set correct_bias=False\nscheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=num_training_steps)  # PyTorch scheduler","metadata":{"execution":{"iopub.status.busy":"2021-06-07T14:07:17.702844Z","iopub.execute_input":"2021-06-07T14:07:17.703160Z","iopub.status.idle":"2021-06-07T14:07:38.643578Z","shell.execute_reply.started":"2021-06-07T14:07:17.703130Z","shell.execute_reply":"2021-06-07T14:07:38.642729Z"},"trusted":true},"execution_count":21,"outputs":[{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/570 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6b63209a09194673a162fc513d644bfb"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/440M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f52ece971af84a83864d7d43f0e1e21b"}},"metadata":{}},{"name":"stderr","text":"Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\nSome weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\nYou should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n","output_type":"stream"}]},{"cell_type":"markdown","source":"# Training & Inference ","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Store our loss and accuracy for plotting\ntrain_loss_set = []\nlearning_rate = []\n\n# Gradients gets accumulated by default\nmodel.zero_grad()\n\n# tnrange is a tqdm wrapper around the normal python range\nfor _ in tnrange(1,epochs+1,desc='Epoch'):\n  print(\"<\" + \"=\"*22 + F\" Epoch {_} \"+ \"=\"*22 + \">\")\n  # Calculate total loss for this epoch\n  batch_loss = 0\n\n  for step, batch in enumerate(train_dataloader):\n    # Set our model to training mode (as opposed to evaluation mode)\n    model.train()\n    \n    # Add batch to GPU\n    batch = tuple(t.to(device) for t in batch)\n    # Unpack the inputs from our dataloader\n    b_input_ids, b_input_mask, b_labels = batch\n\n    # Forward pass\n    outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n    loss = outputs[0]\n    \n    # Backward pass\n    loss.backward()\n    \n    # Clip the norm of the gradients to 1.0\n    # Gradient clipping is not in AdamW anymore\n    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n    \n    # Update parameters and take a step using the computed gradient\n    optimizer.step()\n    \n    # Update learning rate schedule\n    scheduler.step()\n\n    # Clear the previous accumulated gradients\n    optimizer.zero_grad()\n    \n    # Update tracking variables\n    batch_loss += loss.item()\n\n  # Calculate the average loss over the training data.\n  avg_train_loss = batch_loss / len(train_dataloader)\n\n  #store the current learning rate\n  for param_group in optimizer.param_groups:\n    print(\"\\n\\tCurrent Learning rate: \",param_group['lr'])\n    learning_rate.append(param_group['lr'])\n    \n  train_loss_set.append(avg_train_loss)\n  print(F'\\n\\tAverage Training loss: {avg_train_loss}')\n    \n  # Validation\n\n  # Put model in evaluation mode to evaluate loss on the validation set\n  model.eval()\n\n  # Tracking variables \n  eval_accuracy,eval_mcc_accuracy,nb_eval_steps = 0, 0, 0\n\n  # Evaluate data for one epoch\n  for batch in validation_dataloader:\n    # Add batch to GPU\n    batch = tuple(t.to(device) for t in batch)\n    # Unpack the inputs from our dataloader\n    b_input_ids, b_input_mask, b_labels = batch\n    # Telling the model not to compute or store gradients, saving memory and speeding up validation\n    with torch.no_grad():\n      # Forward pass, calculate logit predictions\n      logits = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n    \n    # Move logits and labels to CPU\n    logits = logits[0].to('cpu').numpy()\n    label_ids = b_labels.to('cpu').numpy()\n\n    pred_flat = np.argmax(logits, axis=1).flatten()\n    labels_flat = label_ids.flatten()\n    \n    df_metrics=pd.DataFrame({'Epoch':epochs,'Actual_class':labels_flat,'Predicted_class':pred_flat})\n    \n    tmp_eval_accuracy = accuracy_score(labels_flat,pred_flat)\n    tmp_eval_mcc_accuracy = matthews_corrcoef(labels_flat, pred_flat)\n    \n    eval_accuracy += tmp_eval_accuracy\n    eval_mcc_accuracy += tmp_eval_mcc_accuracy\n    nb_eval_steps += 1\n\n  print(F'\\n\\tValidation Accuracy: {eval_accuracy/nb_eval_steps}')\n  print(F'\\n\\tValidation MCC Accuracy: {eval_mcc_accuracy/nb_eval_steps}')","metadata":{"execution":{"iopub.status.busy":"2021-06-07T14:08:05.243508Z","iopub.execute_input":"2021-06-07T14:08:05.243830Z","iopub.status.idle":"2021-06-07T17:11:41.890327Z","shell.execute_reply.started":"2021-06-07T14:08:05.243801Z","shell.execute_reply":"2021-06-07T17:11:41.889402Z"},"trusted":true},"execution_count":22,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:9: TqdmDeprecationWarning: Please use `tqdm.notebook.trange` instead of `tqdm.tnrange`\n  if __name__ == '__main__':\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Epoch:   0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"95b560e84792474a9701850cf83233ee"}},"metadata":{}},{"name":"stdout","text":"<====================== Epoch 1 ======================>\n\n\tCurrent Learning rate:  1.3333333333333333e-05\n\n\tAverage Training loss: 0.17299689453832293\n\n\tValidation Accuracy: 0.9749387254901961\n\n\tValidation MCC Accuracy: 0.9613292569719919\n<====================== Epoch 2 ======================>\n\n\tCurrent Learning rate:  6.666666666666667e-06\n\n\tAverage Training loss: 0.06296881614167849\n\n\tValidation Accuracy: 0.9831767429193901\n\n\tValidation MCC Accuracy: 0.9739491094919626\n<====================== Epoch 3 ======================>\n\n\tCurrent Learning rate:  0.0\n\n\tAverage Training loss: 0.03844679174764193\n\n\tValidation Accuracy: 0.9859068627450981\n\n\tValidation MCC Accuracy: 0.9779271882799906\n","output_type":"stream"}]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix,classification_report\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    import itertools\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n\n    print(cm)\n\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    fmt = '.2f' if normalize else 'd'\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], fmt),\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    plt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2021-06-07T17:11:48.978240Z","iopub.execute_input":"2021-06-07T17:11:48.978582Z","iopub.status.idle":"2021-06-07T17:11:48.988202Z","shell.execute_reply.started":"2021-06-07T17:11:48.978548Z","shell.execute_reply":"2021-06-07T17:11:48.987106Z"},"trusted":true},"execution_count":23,"outputs":[]},{"cell_type":"code","source":"## emotion labels\nlabel2int = {\n  \"Negative\": 0,\n  \"Neutral\": 1,\n  \"Positive\": 2\n}","metadata":{"execution":{"iopub.status.busy":"2021-06-07T17:11:54.029089Z","iopub.execute_input":"2021-06-07T17:11:54.029400Z","iopub.status.idle":"2021-06-07T17:11:54.033699Z","shell.execute_reply.started":"2021-06-07T17:11:54.029371Z","shell.execute_reply":"2021-06-07T17:11:54.032375Z"},"trusted":true},"execution_count":24,"outputs":[]},{"cell_type":"code","source":"print(classification_report(df_metrics['Actual_class'].values, df_metrics['Predicted_class'].values, target_names=label2int.keys(), digits=len(label2int)))","metadata":{"execution":{"iopub.status.busy":"2021-06-07T17:11:56.797646Z","iopub.execute_input":"2021-06-07T17:11:56.797956Z","iopub.status.idle":"2021-06-07T17:11:56.810875Z","shell.execute_reply.started":"2021-06-07T17:11:56.797928Z","shell.execute_reply":"2021-06-07T17:11:56.810083Z"},"trusted":true},"execution_count":25,"outputs":[{"name":"stdout","text":"              precision    recall  f1-score   support\n\n    Negative      1.000     1.000     1.000         4\n     Neutral      1.000     1.000     1.000         1\n    Positive      1.000     1.000     1.000         4\n\n    accuracy                          1.000         9\n   macro avg      1.000     1.000     1.000         9\nweighted avg      1.000     1.000     1.000         9\n\n","output_type":"stream"}]},{"cell_type":"code","source":"model_save_folder = 'model/'\ntokenizer_save_folder = 'tokenizer/'\n\npath_model = F'/kaggle/working/{model_save_folder}'\npath_tokenizer = F'/kaggle/working/{tokenizer_save_folder}'\n\n##create the dir\n\n!mkdir -p {path_model}\n!mkdir -p {path_tokenizer}\n\n### Now let's save our model and tokenizer to a directory\nmodel.save_pretrained(path_model)\ntokenizer.save_pretrained(path_tokenizer)\n\nmodel_save_name = 'fineTuneModel.pt'\npath = path_model = F'/kaggle/working/{model_save_folder}/{model_save_name}'\ntorch.save(model.state_dict(),path);","metadata":{"execution":{"iopub.status.busy":"2021-06-07T14:02:59.911817Z","iopub.status.idle":"2021-06-07T14:02:59.912369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Conclusion\n\n#### - With Transfer learning approach , We are using pretrained BERT model to classify tweets in the dataset with Negative , Neutral and Positive , Hope you find this kernal as useful \n\n### Kindly upvote if you like it","metadata":{}}]}