Example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports\n",
    "\n",
    "import numpy as np\n",
    "import sys\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import transformers\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from textattack.models.wrappers import HuggingFaceModelWrapper\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import recall_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import our functions\n",
    "sys.path.append('./code')\n",
    "from preprocessing import *\n",
    "from fuzzy_eval import *\n",
    "from systems import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read and preprocess datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('../new_data/fmcg-retail_en_abea_train.csv')\n",
    "test = pd.read_csv('../new_data/fmcg-retail_en_abea_test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Create new columns**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract term\n",
    "train['term'] = train.apply(lambda x: get_term(x['token'], x['aspect_index']), axis=1)\n",
    "test['term'] = test.apply(lambda x: get_term(x['token'], x['aspect_index']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create windows around terms: size 3 and 5\n",
    "# as the list of tokens\n",
    "train['window_3_tokens'] = train.apply(lambda x: window_3(x['token'], x['aspect_index']), axis = 1)\n",
    "train['window_5_tokens'] = train.apply(lambda x: window_5(x['token'], x['aspect_index']), axis = 1)\n",
    "\n",
    "test['window_3_tokens'] = test.apply(lambda x: window_3(x['token'], x['aspect_index']), axis = 1)\n",
    "test['window_5_tokens'] = test.apply(lambda x: window_5(x['token'], x['aspect_index']), axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# merge tokens in the one piece of text\n",
    "train['window_3'] = train['window_3_tokens'].apply(lambda x: ' '.join(x))\n",
    "train['window_5'] = train['window_5_tokens'].apply(lambda x: ' '.join(x))\n",
    "\n",
    "test['window_3'] = test['window_3_tokens'].apply(lambda x: ' '.join(x))\n",
    "test['window_5'] = test['window_5_tokens'].apply(lambda x: ' '.join(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create labels for main classes cathegories \n",
    "train['aspect_MC'] = train['aspect'].apply(lambda x: x.split('_')[0])\n",
    "test['aspect_MC'] = test['aspect'].apply(lambda x: x.split('_')[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create numerical classes\n",
    "le = LabelEncoder()\n",
    "train['Class_aspect_MC'] = le.fit_transform(train['aspect_MC'])\n",
    "test['Class_aspect_MC'] = le.transform(test['aspect_MC'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "le = LabelEncoder()\n",
    "train['Class_aspect'] = le.fit_transform(train['aspect'])\n",
    "test['Class_aspect'] = le.transform(test['aspect'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "le = LabelEncoder()\n",
    "train['Class_sentiment'] = le.fit_transform(train['sentiment'])\n",
    "test['Class_sentiment'] = le.transform(test['sentiment'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "le = LabelEncoder()\n",
    "train['Class_emotion'] = le.fit_transform(train['emotion'])\n",
    "test['Class_emotion'] = le.transform(test['emotion'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Plots**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# build frequency plot\n",
    "train['aspect'].value_counts().plot(kind='bar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['sentiment'].value_counts().plot(kind='bar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['emotion'].value_counts().plot(kind='bar')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Embeddings "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Upload embedding models**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# textattack bert model\n",
    "model_bert = transformers.AutoModelForSequenceClassification.from_pretrained(\"textattack/bert-base-uncased-yelp-polarity\", output_hidden_states = True)\n",
    "tokenizer_bert = transformers.AutoTokenizer.from_pretrained(\"textattack/bert-base-uncased-yelp-polarity\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# textattack albert model\n",
    "model_albert = transformers.AutoModelForSequenceClassification.from_pretrained(\"textattack/albert-base-v2-yelp-polarity\", output_hidden_states = True)\n",
    "tokenizer_albert = transformers.AutoTokenizer.from_pretrained(\"textattack/albert-base-v2-yelp-polarity\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DistilBERT Yelp Review Sentiment model \n",
    "tokenizer = transformers.AutoTokenizer.from_pretrained('distilbert-base-uncased', use_fast=True)\n",
    "model = transformers.TFAutoModel.from_pretrained(\"spentaur/yelp\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Apply them on the train and test datasets**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# apply on the sentence\n",
    "train['sentence_vector_bert'] = train['sentence'].apply(lambda x: get_vector_bert(x, tokenizer_bert, model_bert))\n",
    "train['sentence_vector_albert'] = train['sentence'].apply(lambda x: get_vector_bert(x, tokenizer_albert, model_albert))\n",
    "train['sentence_vector_distilbert'] = train['sentence'].apply(lambda x: get_vector_TFdistilbert_tokens(x, tokenizer, model))\n",
    "\n",
    "test['sentence_vector_bert'] = test['sentence'].apply(lambda x: get_vector_bert(x, tokenizer_bert, model_bert))\n",
    "test['sentence_vector_albert'] = test['sentence'].apply(lambda x: get_vector_bert(x, tokenizer_albert, model_albert))\n",
    "test['sentence_vector_distilbert'] = test['sentence'].apply(lambda x: get_vector_TFdistilbert_tokens(x, tokenizer, model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# apply on the term\n",
    "train['term_vector_bert'] = train['term'].apply(lambda x: get_vector_bert(x, tokenizer_bert, model_bert))\n",
    "train['term_vector_albert'] = train['term'].apply(lambda x: get_vector_bert(x, tokenizer_bert, model_albert))\n",
    "train['term_vector_distilbert'] = train['term'].apply(lambda x: get_vector_TFdistilbert_tokens(x, tokenizer, model))\n",
    "\n",
    "test['term_vector_bert'] = test['term'].apply(lambda x: get_vector_bert(x, tokenizer_bert, model_bert))\n",
    "test['term_vector_albert'] = test['term'].apply(lambda x: get_vector_bert(x, tokenizer_bert, model_albert))\n",
    "test['term_vector_distilbert'] = test['term'].apply(lambda x: get_vector_TFdistilbert_tokens(x, tokenizer, model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# merge them \n",
    "train['merged_vector_bert'] = train.apply(lambda x: x['sentence_vector_bert'] + x['term_vector_bert'], axis = 1)\n",
    "train['merged_vector_albert'] = train.apply(lambda x: x['sentence_vector_albert'] + x['term_vector_albert'], axis = 1)\n",
    "train['merged_vector_distilbert'] = train.apply(lambda x: x['sentence_vector_distilbert'] + x['term_vector_distilbert'], axis = 1)\n",
    "\n",
    "test['merged_vector_bert'] = test.apply(lambda x: x['sentence_vector_bert'] + x['term_vector_bert'], axis = 1)\n",
    "test['merged_vector_albert'] = test.apply(lambda x: x['sentence_vector_albert'] + x['term_vector_albert'], axis = 1)\n",
    "test['merged_vector_distilbert'] = test.apply(lambda x: x['sentence_vector_distilbert'] + x['term_vector_distilbert'], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#apply on the windows: bert and albert\n",
    "train['window_5_bert'] = train['window_5'].apply(lambda x: get_vector_bert(x, tokenizer_bert, model_bert))\n",
    "train['window_3_bert'] = train['window_3'].apply(lambda x: get_vector_bert(x, tokenizer_bert, model_bert))\n",
    "\n",
    "train['window_5_albert'] = train['window_5'].apply(lambda x: get_vector_bert(x, tokenizer_albert, model_albert))\n",
    "train['window_3_albert'] = train['window_5'].apply(lambda x: get_vector_bert(x, tokenizer_albert, model_albert))\n",
    "\n",
    "test['window_5_bert'] = test['window_5'].apply(lambda x: get_vector_bert(x, tokenizer_bert, model_bert))\n",
    "test['window_3_bert'] = test['window_3'].apply(lambda x: get_vector_bert(x, tokenizer_bert, model_bert))\n",
    "\n",
    "test['window_5_albert'] = test['window_5'].apply(lambda x: get_vector_bert(x, tokenizer_albert, model_albert))\n",
    "test['window_3_albert'] = test['window_5'].apply(lambda x: get_vector_bert(x, tokenizer_albert, model_albert))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#apply on the windows: DistilBert\n",
    "train['window_3_distilbert'] = train['window_3'].apply(lambda x: get_vector_TFdistilbert(x, tokenizer, model))\n",
    "train['window_5_distilbert'] = train['window_5'].apply(lambda x: get_vector_TFdistilbert(x, tokenizer, model))\n",
    "\n",
    "train['window_3_tokens_distilbert'] = train['window_3_tokens'].apply(lambda x: get_vector_TFdistilbert_tokens(x, tokenizer, model))\n",
    "train['window_5_tokens_distilbert'] = train['window_5_tokens'].apply(lambda x: get_vector_TFdistilbert_tokens(x, tokenizer, model))\n",
    "\n",
    "test['window_3_distilbert'] = test['window_3'].apply(lambda x: get_vector_TFdistilbert(x, tokenizer, model))\n",
    "test['window_5_distilbert'] = test['window_5'].apply(lambda x: get_vector_TFdistilbert(x, tokenizer, model))\n",
    "\n",
    "test['window_3_tokens_distilbert'] = test['window_3_tokens'].apply(lambda x: get_vector_TFdistilbert_tokens(x, tokenizer, model))\n",
    "test['window_5_tokens_distilbert'] = test['window_5_tokens'].apply(lambda x: get_vector_TFdistilbert_tokens(x, tokenizer, model))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification models tuning for each task"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Aspect"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It can be performed for all aspect classes (Class_aspect) or for the main aspects (Class_aspect_MC)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Define the best k parameter for each text span and choose the best one**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn_list = [1, 3, 5, 7, 9, 13, 17, 21, 25, 29]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**With FRNN OWA method**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "asp_sent_bert_frnnowa = []\n",
    "for NN in nn_list:\n",
    "    res = cross_validation_ensemble_owa(train, ['sentence_vector_bert'], 'Class_aspect', K_fold, [NN], additive(), additive(), 'labels', 'weighted')\n",
    "    asp_sent_bert_frnnowa.append(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "asp_term_bert_frnnowa = []\n",
    "for NN in nn_list:\n",
    "    res = cross_validation_ensemble_owa(train, ['term_vector_bert'], 'Class_aspect', K_fold, [NN], additive(), additive(), 'labels', 'weighted')\n",
    "    asp_term_bert_frnnowa.append(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "asp_merged_bert_frnnowa = []\n",
    "for NN in nn_list:\n",
    "    res = cross_validation_ensemble_owa(train, ['merged_vector_bert'], 'Class_aspect', K_fold, [NN], additive(), additive(), 'labels', 'weighted')\n",
    "    asp_merged_bert_frnnowa.append(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "asp_w5_bert_frnnowa = []\n",
    "for NN in nn_list:\n",
    "    res = cross_validation_ensemble_owa(train, ['window_5_bert'], 'Class_aspect', K_fold, [NN], additive(), additive(), 'labels', 'weighted')\n",
    "    asp_w5_bert_frnnowa.append(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print out the highest scores to choose the best setup for BERT\n",
    "for array in [asp_sent_bert_frnnowa, asp_term_bert_frnnowa, asp_merged_bert_frnnowa, asp_w5_bert_frnnowa]:\n",
    "    print('The highest F1-score: ', max(array), ' with k = ', nn_list[array.index(max(array))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# repeat the same for ALBERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print out the highest scores to choose the best setup for BERT\n",
    "for array in [asp_sent_albert_frnnowa, asp_term_albert_frnnowa, asp_merged_albert_frnnowa, asp_w5_albert_frnnowa]:\n",
    "    print('The highest F1-score: ', max(array), ' with k = ', nn_list[array.index(max(array))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# repeat the same for Distilbert"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print out the highest scores to choose the best setup for BERT\n",
    "for array in [asp_sent_distilbert_frnnowa, asp_term_distilbert_frnnowa, asp_merged_distilbert_frnnowa, asp_w5_distilbert_frnnowa]:\n",
    "    print('The highest F1-score: ', max(array), ' with k = ', nn_list[array.index(max(array))])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**With FROVOCO**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "asp_sent_bert_frovoco = []\n",
    "for NN in nn_list:\n",
    "    res = cross_validation_ensemble_owa(train, ['sentence_vector_bert'], 'Class_aspect', K_fold, [NN], additive(), additive(), 'labels', 'weighted')\n",
    "    asp_sent_bert_frovoco.append(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "asp_term_bert_frovoco = []\n",
    "for NN in nn_list:\n",
    "    res = cross_validation_ensemble_owa(train, ['term_vector_bert'], 'Class_aspect', K_fold, [NN], additive(), additive(), 'labels', 'weighted')\n",
    "    asp_term_bert_frovoco.append(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "asp_merged_bert_frovoco = []\n",
    "for NN in nn_list:\n",
    "    res = cross_validation_ensemble_owa(train, ['merged_vector_bert'], 'Class_aspect', K_fold, [NN], additive(), additive(), 'labels', 'weighted')\n",
    "    asp_merged_bert_frovoco.append(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "asp_window_5_bert_frovoco = []\n",
    "for NN in nn_list:\n",
    "    res = cross_validation_ensemble_owa(train, ['window_5_vector_bert'], 'Class_aspect', K_fold, [NN], additive(), additive(), 'labels', 'weighted')\n",
    "    asp_window_5_bert_frovoco.append(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print out the highest scores to choose the best setup for BERT\n",
    "for array in [asp_sent_bert_frovoco, asp_term_bert_frovoco, asp_merged_bert_frovoco, asp_window_5_bert_frovoco]:\n",
    "    print('The highest F1-score: ', max(array), ' with k = ', nn_list[array.index(max(array))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# choose the best setup for aspect task: text span, parameter k, embedding method, classification method"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Sentiment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Repeat the same for sentiment task (Class_sentiment)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# choose the best setup for sentiment task: text span, parameter k, embedding method, classification method"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Emotion"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Repeat the same for emotion task (Class_emotion)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# choose the best setup for emotion task: text span, parameter k, embedding method, classification method"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1 Two emotion models: positive and negative"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Positive emotions:** joy+anticipation+positive surprise, satisfaction, trust = 3 classes\n",
    "\n",
    "**Negative emotions:** anger, disgust, dissatisfaction, distrust+fear, sadness+negative surprise = 5 classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# form datasets\n",
    "train_pos = train.loc[train['emotion'].isin(['joy','anticipation', 'satisfaction', 'trust'])]\n",
    "train_neg = train.loc[train['emotion'].isin(['anger', 'disgust', 'dissatisfaction', 'distrust', 'fear', 'sadness'])]\n",
    "\n",
    "train_pos_sup = train.loc[(train['emotion'] == 'surprise') & (train['sentiment'].isin(['very_pos', 'pos']))]\n",
    "train_neg_sup = train.loc[(train['emotion'] == 'surprise') & (train['sentiment'].isin(['very_neg', 'neg']))]\n",
    "\n",
    "train_pos = pd.concat([train_pos, train_pos_sup])\n",
    "train_neg = pd.concat([train_neg, train_neg_sup])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for positive emotion\n",
    "train_pos['pos_emotion'] = None\n",
    "\n",
    "for i in train_pos.index.to_list():\n",
    "    if train_pos['emotion'][i] in ['joy','anticipation', 'surprise']:\n",
    "        train_pos['pos_emotion'][i] = 'JAS'\n",
    "    elif train_pos['emotion'][i]=='satisfaction':\n",
    "        train_pos['pos_emotion'][i] = 'S'\n",
    "    elif train_pos['emotion'][i]=='trust':\n",
    "        train_pos['pos_emotion'][i] = 'T'\n",
    "    else:\n",
    "        print(train_pos['emotion'][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for negative emotions\n",
    "train_neg['neg_emotion'] = None\n",
    "\n",
    "for i in train_neg.index.to_list():\n",
    "    if train_neg['emotion'][i] in ['distrust', 'fear']:\n",
    "        train_neg['neg_emotion'][i] = 'DF'\n",
    "    elif train_neg['emotion'][i] in ['sadness', 'surprise']:\n",
    "        train_neg['neg_emotion'][i] = 'SS'\n",
    "    elif train_neg['emotion'][i]=='anger':\n",
    "        train_neg['neg_emotion'][i] = 'A'\n",
    "    elif train_neg['emotion'][i]=='disgust':\n",
    "        train_neg['neg_emotion'][i] = 'DT'\n",
    "    elif train_neg['emotion'][i]=='dissatisfaction':\n",
    "        train_neg['neg_emotion'][i] = 'DN'\n",
    "    else:\n",
    "        print(train_neg['emotion'][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# form classes\n",
    "\n",
    "le = LabelEncoder()\n",
    "train_neg['Class_neg_emotion'] = le.fit_transform(train_neg['neg_emotion'])\n",
    "test_neg['Class_neg_emotion'] = le.transform(test_neg['neg_emotion'])\n",
    "\n",
    "le = LabelEncoder()\n",
    "train_pos['Class_pos_emotion'] = le.fit_transform(train_pos['pos_emotion'])\n",
    "test_pos['Class_pos_emotion'] = le.transform(test_pos['pos_emotion'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# repeat same experiments\n",
    "# choose the best setup for positive emotion task: text span, parameter k, embedding method, classification method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# choose the best setup for negative emotion task: text span, parameter k, embedding method, classification method"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Systems"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use the best setups from the above."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Basic pipeline, where aspect, sentiment, and emotion tasks are performed one after one\n",
    "\n",
    "#[vector_name_asp, vector_name_sen, vector_name_emo]: list of strings, which represent name of columns in train (test) dataset with feature vectors that we will use for aspect/sentiment/emotion tasks \n",
    "#[k_asp, k_sen, k_emo]: list of integers, which represents parameter k (a number of neighbours) that we will use for aspect/sentiment/emotion tasks \n",
    "#they should be obtained from the previous model tuning experiments\n",
    "\n",
    "res_asp, res_sent, res_emo = system_0(train, test, ['Class_aspect', 'Class_sentiment', 'Class_emotion'], \n",
    "                                      [vector_name_asp, vector_name_sen, vector_name_emo], [k_asp, k_sen, k_emo])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# System #1,  where as class_aspect we should use main aspect classes and for emotions we created two modes: one for positive emotions and one for negative\n",
    "\n",
    "#[vector_name_asp, vector_name_sen, vector_name_emo_pos, vector_name_emo_neg]: list of strings, which represent name of columns in train (test) dataset with feature vectors that we will use for aspect/sentiment/emotion tasks \n",
    "#[k_asp, k_sen, k_emo_pos, k_emo_neg]: list of integers, which represents parameter k (a number of neighbours) that we will use for aspect/sentiment/emotion tasks \n",
    "#they should be obtained from the previous model tuning experiments\n",
    "\n",
    "res_asp, res_sent, res_emo = system_1(train, test, ['Class_aspect', 'Class_sentiment', 'Class_pos_emotion', 'Class_neg_emotion'], \n",
    "                                      [vector_name_asp, vector_name_sen, vector_name_emo_pos, vector_name_emo_neg], \n",
    "                                      [k_asp, k_sen, k_emo_pos, k_emo_neg])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# System #2, where for results of sentiment task we perform filtration with a usage of the cost scores. \n",
    "\n",
    "#[vector_name_asp, vector_name_sen, vector_name_emo_pos, vector_name_emo_neg]: list of strings, which represent name of columns in train (test) dataset with feature vectors that we will use for aspect/sentiment/emotion tasks \n",
    "#[k_asp, k_sen, k_emo_pos, k_emo_neg]: list of integers, which represents parameter k (a number of neighbours) that we will use for aspect/sentiment/emotion tasks \n",
    "#they should be obtained from the previous model tuning experiments\n",
    "\n",
    "res_asp, res_sent, res_emo = system_2(train, test, ['Class_aspect', 'Class_sentiment', 'Class_pos_emotion', 'Class_neg_emotion'], \n",
    "                                      [vector_name_asp, vector_name_sen, vector_name_emo_pos, vector_name_emo_neg], \n",
    "                                      [k_asp, k_sen, k_emo_pos, k_emo_neg], 'data/sentiment_cost.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# System #3, where all tasks (aspect, sentiment, emotion) are performed separately \n",
    "\n",
    "#[vector_name_asp, vector_name_sen, vector_name_emo_pos, vector_name_emo_neg]: list of strings, which represent name of columns in train (test) dataset with feature vectors that we will use for aspect/sentiment/emotion tasks \n",
    "#[k_asp, k_sen, k_emo_pos, k_emo_neg]: list of integers, which represents parameter k (a number of neighbours) that we will use for aspect/sentiment/emotion tasks \n",
    "#they should be obtained from the previous model tuning experiments\n",
    "\n",
    "res_asp, res_sent, res_emo = system_2(train, test, ['Class_aspect', 'Class_sentiment', 'Class_pos_emotion', 'Class_neg_emotion'], \n",
    "                                      [vector_name_asp, vector_name_sen, vector_name_emo_pos, vector_name_emo_neg], \n",
    "                                      [k_asp, k_sen, k_emo_pos, k_emo_neg])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#use predictions to create new columns\n",
    "\n",
    "test['Predicted_asp_label'] = res_asp\n",
    "test['Predicted_sent_label'] = res_sent\n",
    "test['Predicted_emo_label'] = res_emo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**With F1-score**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f1_score(test[\"Class_aspect_MC\"].to_list(), test['Predicted_asp_label'].to_list(), average = \"weighted\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f1_score(test[\"Class_sentiment\"].to_list(), test['Predicted_sent_label'].to_list(), average = \"weighted\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f1_score(test[\"Class_emotion\"].to_list(), test['Predicted_emo_label'].to_list(), average = \"weighted\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**With Cost Corrected Accuracy**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for sentiment\n",
    "cf = confusion_matrix(test[\"Class_sentiment\"].to_list(), test['Predicted_sen_label'].to_list())\n",
    "ct_pol_path = 'data/sentiment_cost.json'\n",
    "pol_labels = [\"neg\", \"neu\", \"pos\", \"very_neg\", \"very_pos\", \"y\"]\n",
    "calculate_cost_corrected_accuracy(pol_labels, cf, ct_pol_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for emotion\n",
    "cf = confusion_matrix(test[\"Class_emotion\"].to_list(), test['Predicted_emo_label'].to_list())\n",
    "ct_emo_path = 'data/emotion_cost.json'\n",
    "emo_labels = [\"anger\",\"anticipation\",\"disgust\",\"dissatisfaction\",\"distrust\",\"fear\",\"joy\",\"neutral\",\"sadness\",\"satisfaction\",\n",
    "              \"surprise\",\"trust\",\"y\"]\n",
    "calculate_cost_corrected_accuracy(emo_labels, cf, ct_emo_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Explore the closest neighbours for the test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "i = #choose id number to check\n",
    "print(test['term'][i])\n",
    "print(test['sentence'][i])\n",
    "print(test['aspect'][i])\n",
    "print(test['aspect_MC'][i])\n",
    "print(test['sentiment'][i])\n",
    "print(test['emotion'][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vector_name = #name of column to evaluate, for example, 'merged_vector_distilbert'\n",
    "k = #number of neighbours to consider, for example, 5\n",
    "text_span = #name of text span to use, for example, 'sentence'\n",
    "class_name = #name of class to evaluate, for example, 'aspect_MC'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_neigbours(test[vector_name][i], train, vector_name, k, text_span, class_name)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}