TweetsSuicide_6_classes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "import emoji\n",
    "import ktrain\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from eli5.lime import TextExplainer\n",
    "from matplotlib import pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from twitterFunctions.cross_validation import run_SVM_CV\n",
    "from twitterFunctions.performanceMetrics import get_performance\n",
    "from twitterFunctions.processing import (\n",
    "    fix_emotes,\n",
    "    process_token_fin,\n",
    "    process_tweet,\n",
    ")\n",
    "from twitterFunctions.training import predict_test, set_seeds, train_learner\n",
    "\n",
    "\n",
    "plt.rcdefaults()\n",
    "\n",
    "\n",
    "SEED = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load training data\n",
    "file = \"./data/training_posts20201201_main_categories.tsv\"\n",
    "# read training data\n",
    "df = pd.read_csv(file, sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter relevant columns\n",
    "df = df[[\"notserious_unclear\", \"focus\", \"type\", \"main_category\", \"contents\"]]\n",
    "# rename class and text columns\n",
    "colNames = [\"notserious_unclear\", \"focus\", \"type\", \"class\", \"text\"]\n",
    "df.columns = colNames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# here we print the class frequency distributions\n",
    "freq_combined = Counter(df[\"class\"].values)\n",
    "objects = []\n",
    "values = []\n",
    "for i in freq_combined.keys():\n",
    "    objects.append(i)\n",
    "for i in freq_combined.values():\n",
    "    values.append(i)\n",
    "\n",
    "y_pos = np.arange(len(objects))\n",
    "\n",
    "plt.bar(y_pos, values, align='center', alpha=0.5)\n",
    "plt.xticks(y_pos, objects)\n",
    "plt.ylabel('Frequency ')\n",
    "plt.title('Class distribution tweets')\n",
    "plt.xticks(rotation=90)\n",
    "plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we run the preprocessing from the /twitterFunctions/processing.py\n",
    "# here we use 3 functions (each loops over the entire dataset, they have to run sequentailly)\n",
    "# if it takes too long one can make one function with all preprocessing - this recudes the runtime significantly\n",
    "temp = df.text.apply(process_tweet)\n",
    "# emojy mapping\n",
    "emojis = temp.apply(emoji.emojize)\n",
    "emojis = emojis.apply(fix_emotes)\n",
    "cleaned_text = emojis\n",
    "X_train_str = process_token_fin(cleaned_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train val and test split\n",
    "# stratify=df[\"class\"] ensures that the subsets contain a similar distributions as original data\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X_train_str, df[\"class\"], test_size=0.2, random_state=1, stratify=df[\"class\"])\n",
    "\n",
    "X_train, X_val, y_train, y_val = train_test_split(\n",
    "    X_train, y_train, test_size=0.2, random_state=1, stratify=y_train)\n",
    "\n",
    "class_names = set(y_train.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# visual analysis of number of tokens\n",
    "ad = []\n",
    "for i in X_train:\n",
    "    ad.append(len(i.split(\" \")))\n",
    "\n",
    "_ = plt.hist(ad, bins='auto')\n",
    "plt.xlabel(\"# of tokens\")\n",
    "plt.ylabel(\"frequency\")\n",
    "plt.title(\"Train Set: sequence length histogram\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Majority Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "majority_class = Counter(y_train).most_common()[0][0]\n",
    "y_val_predicted = np.repeat(majority_class, len(y_val))\n",
    "y_test_predicted = np.repeat(majority_class, len(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_results = get_performance(\n",
    "    y_val, y_val_predicted, list(Counter(y_train).keys()))\n",
    "test_results = get_performance(\n",
    "    y_test, y_test_predicted, list(Counter(y_train).keys()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TFIDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Validation Performance: \\n\")\n",
    "model = run_SVM_CV(X_train, X_val, y_train, y_val)\n",
    "pred = model.predict(X_test)\n",
    "print(\"Test Performance: \\n\")\n",
    "mat = get_performance(y_test.values, pred, list(class_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training BERT / XLNET model\n",
    "Set the *model_name* flag in the *train_learner* function to the model you want to train. For BERT use **bert-base-uncased**, for XLNET use **xlnet-base-cased**."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "original_learner = train_learner(\n",
    "    X_train, y_train.values,\n",
    "    X_val, y_val.values,\n",
    "    lr=3e-5, epoch=5, seed=1, text_length=80,  # parameters for the training\n",
    "    #   checkpoint_folder=\"D:/models/test/\",   # add the path where the checkpoints should be saved\n",
    "    model_name=\"bert-base-uncased\"  # xlnet-base-cased\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## After the training has finished, you will see the training statistics. I usually choose the epoch with the highest validation accuracy (if two performances were similar I chose the one with the lower validation loss).\n",
    "\n",
    "If you used a checkpoint folder, you can load the model with the best performance rather than the last epoch. E.g. if you trained for 5 epochs, the default behaviour is that the model will use the last run, i.e. epoch 5. But if the performance of epoch 5 is worse than epoch 3, you can manually load the weights of epoch 3. See below:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# original_learner[4].load_weights(\"D:/models/test/weights-01.hdf5\") # your path + weights-01.hdf5\n",
    "# where weights-*BEST_EPOCH*.hdf5\n",
    "learner_reloaded = ktrain.get_learner(\n",
    "    original_learner[4], train_data=original_learner[2], val_data=original_learner[3], batch_size=2)\n",
    "\n",
    "model_ = learner_reloaded\n",
    "t_ = original_learner[1]\n",
    "trn_ = original_learner[2]\n",
    "# setting seeds (redundantly) before predicting validation set\n",
    "set_seeds(SEED)\n",
    "classNames2 = np.unique(y_train)\n",
    "# usually you only have to do this if you are interested in inter-class statistics of the validation set:D\n",
    "# PREDICT ON VALIDATION SET\n",
    "pred = predict_test(X_val, model_,\n",
    "                    t=t_,\n",
    "                    trn=trn_)\n",
    "predictor = pred[1]\n",
    "val = t_.preprocess_test(X_val, y_val.values)\n",
    "model_.validate(val_data=val)\n",
    "mat = get_performance(y_val.values, pred[0], classNames2)\n",
    "\n",
    "# predict TEST set, this one you must always do!\n",
    "set_seeds(SEED)\n",
    "pred = predict_test(X_test, model_,\n",
    "                    t=t_,\n",
    "                    trn=trn_)\n",
    "predictor = pred[1]\n",
    "test = t_.preprocess_test(X_test, y_test.values)\n",
    "model_.validate(val_data=test, class_names=list(classNames2))\n",
    "mat = get_performance(y_test.values, pred[0], classNames2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving a ktrain model\n",
    "\n",
    "If the test performance was good, and you want to save the actual model (not only the weights of the training) call **predictor.save( \"yourPath\")**."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# predictor.save('D:/models/Twitter_6_final')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this prints the confusion matrix for the test set and its predictions\n",
    "\n",
    "mat = get_performance(y_test.values, pred, classNames2)\n",
    "\n",
    "labels = y_test.values\n",
    "capitalized = [i.capitalize() for i in classNames2]\n",
    "index = columns = capitalized\n",
    "cm_df = pd.DataFrame(mat/np.sum(mat), columns, index)\n",
    "plt.figure(figsize=(10, 8.2))\n",
    "sns.heatmap(cm_df, annot=True, fmt=\".0%\"\n",
    "            )\n",
    "plt.yticks(rotation=0, fontsize=12)\n",
    "plt.xlabel(\"Predicted class\", fontsize=20)\n",
    "plt.ylabel(\"Original class\", fontsize=20)\n",
    "plt.title(\"Confusion matrix test set, n=641\", fontsize=25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Playing around with ELI5 text explainer to make the black box models a little more expalinable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = pd.DataFrame({\n",
    "    \"text\": X_test,\n",
    "    \"true_label\": y_test,\n",
    "    \"predicted_label\": pred[0]\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# select a specific test sample you want to investiage more clsoely\n",
    "doc = X_test[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# here we train a text explainer that runs n_samples to simualte the model behaviour\n",
    "te = TextExplainer(random_state=SEED, n_samples=20)\n",
    "_ = te.fit(doc, predictor.predict_proba)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = te.show_weights(target_names=predictor.c, targets=pred[0][150:155], top=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "te.show_weights()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}