preprocessing_embedding_baseline_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "68170d4d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: gensim in c:\\programdata\\anaconda3\\envs\\ml\\lib\\site-packages (4.1.2)\n",
      "Requirement already satisfied: scipy>=0.18.1 in c:\\programdata\\anaconda3\\envs\\ml\\lib\\site-packages (from gensim) (1.7.1)\n",
      "Requirement already satisfied: Cython==0.29.23 in c:\\programdata\\anaconda3\\envs\\ml\\lib\\site-packages (from gensim) (0.29.23)\n",
      "Requirement already satisfied: numpy>=1.17.0 in c:\\programdata\\anaconda3\\envs\\ml\\lib\\site-packages (from gensim) (1.21.2)\n",
      "Requirement already satisfied: smart-open>=1.8.1 in c:\\programdata\\anaconda3\\envs\\ml\\lib\\site-packages (from gensim) (5.2.1)\n"
     ]
    }
   ],
   "source": [
    "!pip install gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "e216230e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from baseline import *\n",
    "from embeddings import * \n",
    "from preprocessing import * \n",
    "from submission import *\n",
    "from helpers import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "7e9ec53f",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a862539a",
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH_DATA = './data/'\n",
    "PATH_PREPROCESSING = PATH_DATA + 'preprocessing/'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd9827eb",
   "metadata": {},
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a5c5ae10",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_pos, train_neg = load_tweets(PATH_DATA, small_dataset=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "76bccd78",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "file saved\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\Younes\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package webtext to\n",
      "[nltk_data]     C:\\Users\\Younes\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package webtext is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# preparing files used in the preprocessing\n",
    "# slang \n",
    "lines = slang_dict_to_tuple(PATH_PREPROCESSING)\n",
    "slang_dict = slang_tuple_to_dict(lines)\n",
    "slang_tuple_to_json(slang_dict,PATH_PREPROCESSING )\n",
    "slang_list = json.loads(open(PATH_PREPROCESSING + 'slang.json', 'r').read())\n",
    "# stopwords vocab\n",
    "nltk.download('stopwords')\n",
    "# tokenizer used to remove punctuation\n",
    "tokenizer = nltk.RegexpTokenizer(r\"\\w+\")\n",
    "# Remove words with low occurences \n",
    "nltk.download('webtext') #https://www.pythonprogramming.in/find-frequency-of-each-word-from-a-text-file-using-nltk.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7fbb9c2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# counting the words in the train_pos and train_neg dataframes and saving them to a text file\n",
    "# beware to produce the file for the correct ( full or small) dataset before using them later\n",
    "word_occ_tofile(PATH_PREPROCESSING, file_name = 'count_word_pos.txt', df = train_pos )\n",
    "word_occ_tofile(PATH_PREPROCESSING, file_name = 'count_word_neg.txt', df = train_neg )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b1b8d030",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\Younes\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "# loading some files required for the preprocessing\n",
    "contraction_list = load_contractions(PATH_PREPROCESSING)\n",
    "stemmer = SnowballStemmer(\"english\")\n",
    "all_stopwords = load_stopwords()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "25388623",
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_words_pos = low_occuring_words(PATH_PREPROCESSING, file_name = 'count_word_pos.txt' )\n",
    "filter_words_neg = low_occuring_words(PATH_PREPROCESSING, file_name = 'count_word_neg.txt' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "44f8600b",
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_words_all = filter_words_pos + filter_words_neg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "173a88f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = pd.read_fwf(PATH_DATA + 'twitter-datasets/test_data.txt', header = None, names = ['Tweet'], colspecs = [(0,280)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "9f310756",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████| 100000/100000 [00:00<00:00, 869951.11it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████| 100000/100000 [00:00<00:00, 433156.26it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████| 100000/100000 [00:00<00:00, 418624.97it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████| 100000/100000 [00:00<00:00, 901244.98it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████| 100000/100000 [00:00<00:00, 162955.39it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████| 100000/100000 [00:07<00:00, 13956.82it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████| 100000/100000 [00:00<00:00, 272132.27it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████| 100000/100000 [00:00<00:00, 423971.56it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████| 100000/100000 [00:00<00:00, 971294.92it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████| 100000/100000 [00:00<00:00, 392354.41it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████| 100000/100000 [00:00<00:00, 362546.18it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████| 100000/100000 [00:00<00:00, 877588.79it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████| 100000/100000 [00:00<00:00, 167877.25it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████| 100000/100000 [00:09<00:00, 10416.09it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████| 100000/100000 [00:00<00:00, 226886.66it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████| 100000/100000 [00:00<00:00, 342662.66it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 833940.55it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 238250.46it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 232693.70it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 833890.81it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 137028.46it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████| 10000/10000 [00:01<00:00, 7728.10it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 192424.86it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 303210.01it/s]\n"
     ]
    }
   ],
   "source": [
    "train_pos = preprocessing(train_pos, contraction_list, stemmer, \n",
    "                          filter_words_all, slang_list, all_stopwords, tokenizer)\n",
    "train_neg = preprocessing(train_neg, contraction_list, stemmer, \n",
    "                          filter_words_all, slang_list, all_stopwords, tokenizer) \n",
    "test_data = preprocessing(test_data, contraction_list, stemmer, \n",
    "                          filter_words_all, slang_list, all_stopwords, tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddce2922",
   "metadata": {},
   "source": [
    "# Embeddings"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8636570f",
   "metadata": {},
   "source": [
    "## Glove"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "63a8fc2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_model = load_glove_model(path_pretrained_embeddings = 'data/twitter-datasets/glove-twitter-25.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "138a1066",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████| 100000/100000 [00:03<00:00, 30648.51it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████| 100000/100000 [00:03<00:00, 26543.98it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 26470.17it/s]\n"
     ]
    }
   ],
   "source": [
    "train_pos_embeddings = df_to_GloVe(train_pos, model1=glove_model)\n",
    "train_neg_embeddings = df_to_GloVe(train_neg, model1=glove_model)\n",
    "test_data_embeddings = df_to_GloVe(test_data, model1=glove_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "4ce137ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_cols(train_pos_embeddings)\n",
    "clean_cols(train_neg_embeddings)\n",
    "clean_cols(test_data_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "8a3069a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "x, y = df_to_numpy_features(train_pos_embeddings,train_neg_embeddings)\n",
    "x, means,stds = standardize_cols(x, mean_x=None, std_x=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "de86b062",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_d = test_data_embeddings.to_numpy(copy=True)\n",
    "test_x, _, _ = standardize_cols(test_d, means, stds)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5d01d9f",
   "metadata": {},
   "source": [
    "## TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "517d2459",
   "metadata": {},
   "outputs": [],
   "source": [
    "vectors, vectorizer = tf_idf_embedding(train_pos, train_neg, max_features=10000, ngram_range=(1,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "68315391",
   "metadata": {},
   "outputs": [],
   "source": [
    "x, y = add_label_tfidf(vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "45bfd6cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<200000x9121 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 1076769 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "461fcd9a",
   "metadata": {},
   "source": [
    "# Baseline models"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "548fcb15",
   "metadata": {},
   "source": [
    "## Naive bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d6814eeb",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "naive_bayes_cv(x, y, splits=5, glove=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "af767483",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.750245"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gnb = naive_bayes(x, y, glove=False)\n",
    "gnb.score(x,y)  # cell was run with tf-idf embeddings"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52c95699",
   "metadata": {},
   "source": [
    "# Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7367080f",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "logistic_regression_cv(x, y, \n",
    "                       solvers = ['lbfgs', 'saga'],\n",
    "                       penalty = ['l2'], \n",
    "                       c_values = [100, 10, 1.0, 0.1, 0.01], \n",
    "                       splits = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "d0cc99a3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.78546"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr = logistic_regression(x, y, solver='saga', penalty='l2', c_value=1.0)\n",
    "lr.score(x,y) # cell was run with tf-idf embeddings"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cda70486",
   "metadata": {},
   "source": [
    "# MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1846bb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp_cv(x, y, \n",
    "       solvers =['adam'], \n",
    "       lrs = [0.1, 0.01,0.001], \n",
    "       activations = ['tanh','relu','logistic'],\n",
    "       max_iters=[10,50], splits = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "42d6311c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.640955"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlp1 = mlp(x, y, solver='adam', lr=0.001, act='tanh', max_iters=50)\n",
    "mlp1.score(x,y) # cell was run with glove embeddings"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}