preprocessing_data_and_clustering_tweet_topics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "bHGihPrKMAJa",
    "outputId": "e7da904c-4d0d-42d6-83ee-bab85a52d5a6"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nejGa1C50IrM"
   },
   "outputs": [],
   "source": [
    "\n",
    "#from preprocessing import tokenize, export_to_csv\n",
    "from gsdmm import MovieGroupProcess\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pytorch-pretrained-bert in /home/skarra7/anaconda3/lib/python3.8/site-packages (0.6.2)\n",
      "Requirement already satisfied: regex in /home/skarra7/anaconda3/lib/python3.8/site-packages (from pytorch-pretrained-bert) (2020.6.8)\n",
      "Requirement already satisfied: numpy in /home/skarra7/anaconda3/lib/python3.8/site-packages (from pytorch-pretrained-bert) (1.18.5)\n",
      "Requirement already satisfied: requests in /home/skarra7/anaconda3/lib/python3.8/site-packages (from pytorch-pretrained-bert) (2.24.0)\n",
      "Requirement already satisfied: torch>=0.4.1 in /home/skarra7/anaconda3/lib/python3.8/site-packages (from pytorch-pretrained-bert) (1.6.0)\n",
      "Requirement already satisfied: boto3 in /home/skarra7/anaconda3/lib/python3.8/site-packages (from pytorch-pretrained-bert) (1.17.102)\n",
      "Requirement already satisfied: tqdm in /home/skarra7/anaconda3/lib/python3.8/site-packages (from pytorch-pretrained-bert) (4.47.0)\n",
      "Requirement already satisfied: future in /home/skarra7/anaconda3/lib/python3.8/site-packages (from torch>=0.4.1->pytorch-pretrained-bert) (0.18.2)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /home/skarra7/anaconda3/lib/python3.8/site-packages (from boto3->pytorch-pretrained-bert) (0.10.0)\n",
      "Requirement already satisfied: s3transfer<0.5.0,>=0.4.0 in /home/skarra7/anaconda3/lib/python3.8/site-packages (from boto3->pytorch-pretrained-bert) (0.4.2)\n",
      "Requirement already satisfied: botocore<1.21.0,>=1.20.102 in /home/skarra7/anaconda3/lib/python3.8/site-packages (from boto3->pytorch-pretrained-bert) (1.20.102)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /home/skarra7/anaconda3/lib/python3.8/site-packages (from botocore<1.21.0,>=1.20.102->boto3->pytorch-pretrained-bert) (1.25.9)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /home/skarra7/anaconda3/lib/python3.8/site-packages (from botocore<1.21.0,>=1.20.102->boto3->pytorch-pretrained-bert) (2.8.1)\n",
      "Requirement already satisfied: six>=1.5 in /home/skarra7/anaconda3/lib/python3.8/site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.21.0,>=1.20.102->boto3->pytorch-pretrained-bert) (1.15.0)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /home/skarra7/anaconda3/lib/python3.8/site-packages (from requests->pytorch-pretrained-bert) (2.10)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/skarra7/anaconda3/lib/python3.8/site-packages (from requests->pytorch-pretrained-bert) (2020.6.20)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /home/skarra7/anaconda3/lib/python3.8/site-packages (from requests->pytorch-pretrained-bert) (3.0.4)\n"
     ]
    }
   ],
   "source": [
    "!pip install pytorch-pretrained-bert\n",
    "from pytorch_pretrained_bert import BertTokenizer, BertModel, BertForMaskedLM\n",
    "from IPython.display import clear_output\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased',do_lower_case=False)\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /home/skarra7/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.corpus import stopwords\n",
    "nltk.download('stopwords')\n",
    "\n",
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stopword(text):\n",
    "    text = text.lower().split()\n",
    "    stops = set(stopwords.words(\"english\"))\n",
    "    text = [w for w in text if not w in stops]\n",
    "    text = \" \".join(text)\n",
    "    return text\n",
    "\n",
    "def stemwords(text):    \n",
    "    ps = PorterStemmer()\n",
    "    text = text.lower().split()\n",
    "    stemmed_words = [ps.stem(word) for word in text]\n",
    "    text = \" \".join(stemmed_words)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pip in /home/skarra7/anaconda3/lib/python3.8/site-packages (21.1.3)\n",
      "Requirement already satisfied: setuptools in /home/skarra7/anaconda3/lib/python3.8/site-packages (57.0.0)\n",
      "Requirement already satisfied: wheel in /home/skarra7/anaconda3/lib/python3.8/site-packages (0.36.2)\n"
     ]
    }
   ],
   "source": [
    "!pip install -U pip setuptools wheel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This pickle file is obatined by filtering english language users who have atleast 20 tweets on their home timeline\n",
    "#It is straightforward hence we generate a pickle file of filtered dataset and use here directly.\n",
    "\n",
    "file = open(\"filtered_english.pkl\",'rb')\n",
    "df = pickle.load(file)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- 50.20185446739197 seconds ---\n",
      "--- 72.20108079910278 seconds ---\n"
     ]
    }
   ],
   "source": [
    "nlp = spacy.load('en_core_web_sm',disable = ['parser','entity'])\n",
    "import time\n",
    "start_time = time.time()\n",
    "\n",
    "file = open(\"filtered_english.pkl\",'rb')\n",
    "df = pickle.load(file)\n",
    "file.close()\n",
    "\n",
    "df.drop([\"engaging_user_id\",\"tweet_id\",\"tweet_timestamp\",\"reply_timestamp\",\"retweet_timestamp\",\"retweet_with_comment_timestamp\",\"like_timestamp\"], axis = 1)\n",
    "df['text_tokens']= df['text_tokens'].str.split('\\t').apply(lambda x:list(map(int, x)))\n",
    "print (\"--- %s seconds ---\" % (time.time() - start_time))  \n",
    "\n",
    "df['text_tokens'] = df['text_tokens'].apply(lambda x:tokenizer.convert_ids_to_tokens(x))\n",
    "print (\"--- %s seconds ---\" % (time.time() - start_time))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def conv(s):\n",
    "    l = []\n",
    "    for i in range(len(s)):\n",
    "        if s[i][:2]!='##':\n",
    "            l.append(s[i])\n",
    "        else:\n",
    "            l[-1]=l[-1]+s[i][2:]\n",
    "    return l\n",
    "\n",
    "\n",
    "df['text_tokens'] = df['text_tokens'].apply(lambda x: conv(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "57    [[CLS], Even, by, the, standards, of, Trump, a...\n",
       "58    [[CLS], EXCLUSIVE, :, Manuscript, of, Bolton, ...\n",
       "61    [[CLS], Whoever, the, patriot, is, that, leake...\n",
       "62    [[CLS], If, anything, the, president, does, to...\n",
       "70    [[CLS], Flood, my, guts, ., ., #, bigdick, #, ...\n",
       "Name: text_tokens, dtype: object"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['text_tokens'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- 106.4303925037384 seconds ---\n",
      "--- 110.95485138893127 seconds ---\n",
      "--- 152.6514790058136 seconds ---\n",
      "--- 803.4099097251892 seconds ---\n",
      "--- 1039.5841271877289 seconds ---\n"
     ]
    }
   ],
   "source": [
    "df['text_tokens']= df['text_tokens'].apply(lambda x: \" \".join([y for y in x]))\n",
    "print (\"--- %s seconds ---\" % (time.time() - start_time))  \n",
    "\n",
    "for r in ((\"[CLS]\", \"\"),(\"[SEP]\",\"\"),(\"[UNK]\",\"\"),(\"RT\",\"\")):\n",
    "    df['text_tokens'] = df['text_tokens'].apply(lambda x: x.replace(*r))\n",
    "print (\"--- %s seconds ---\" % (time.time() - start_time))  \n",
    "\n",
    "df[\"text_tokens\"] = df[\"text_tokens\"].apply(lambda x:' '.join(re.sub(\"(@[A-Za-z0-9]+)|([^0-9A-Za-z \\t])|(\\w+:\\/\\/\\S+)\",\" \",x).split()))\n",
    "print (\"--- %s seconds ---\" % (time.time() - start_time))  \n",
    "\n",
    "df[\"text_tokens\"] = df[\"text_tokens\"].apply(lambda x: stemwords(x))\n",
    "print (\"--- %s seconds ---\" % (time.time() - start_time))\n",
    "\n",
    "df[\"text_tokens\"] = df[\"text_tokens\"].apply(lambda x: stopword(x))\n",
    "print (\"--- %s seconds ---\" % (time.time() - start_time))  \n",
    "\n",
    "\n",
    "#df.to_pickle('text_for_topics_1.pkl')\n",
    " \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "57    even standard trump apologist believ anyth exc...\n",
       "58    exclus manuscript bolton book say told ag barr...\n",
       "61         whoever patriot leak bolton manuscript thank\n",
       "62    anyth presid doe get elect ok could take step ...\n",
       "70    flood gut bigdick breed bareback raw repost ju...\n",
       "Name: text_tokens, dtype: object"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['text_tokens'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['text_tokens'] = df['text_tokens'].apply(lambda x: x.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Voc size: 1361455\n",
      "Number of documents: 3409013\n"
     ]
    }
   ],
   "source": [
    "docs = df['text_tokens'].tolist()\n",
    "vocab = set(x for doc in docs for x in doc)\n",
    "n_terms = len(vocab)\n",
    "\n",
    "print(\"Voc size:\", n_terms)\n",
    "print(\"Number of documents:\", len(docs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "mgp = MovieGroupProcess(K=100, alpha=0.1, beta=0.1, n_iters=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train a new model \n",
    "\n",
    "# Init of the Gibbs Sampling Dirichlet Mixture Model algorithm\n",
    "\n",
    "start_time = time.time()\n",
    "vocab = set(x for doc in docs for x in doc)\n",
    "n_terms = len(vocab)\n",
    "n_docs = len(docs)\n",
    "\n",
    "# Fit the model on the data given the chosen seeds\n",
    "y = mgp.fit(docs, n_terms)\n",
    "print (\"--- %s seconds ---\" % (time.time() - start_time))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_count = np.array(mgp.cluster_doc_count)\n",
    "print('Number of documents per topics :', sorted(doc_count))\n",
    "print('*'*20)\n",
    "\n",
    "# Topics sorted by document inside\n",
    "top_index = doc_count.argsort()[-10:][::-1]\n",
    "print('Most important clusters (by number of docs inside):', top_index)\n",
    "print('*'*20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def top_words(distribution, top_index, num_words):\n",
    "    for topic in top_index:\n",
    "        pairs = sorted([(k, v) for k, v in distribution[topic].items()], key=lambda x: x[1], reverse=True)\n",
    "        a = pairs[:num_words]\n",
    "        c = [i[0] for i in a]\n",
    "        print (c)\n",
    "        print('.'*30)\n",
    "\n",
    "b = top_words(mgp.cluster_word_distribution, top_index, 1000)        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['text_tokens'] = df['text_tokens'].apply(lambda x: mgp.choose_best_label(x)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('gsdmm_topics.pkl', \"wb\") as f:\n",
    "     pickle.dump(df, f)\n",
    "     f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "TPU",
  "colab": {
   "machine_shape": "hm",
   "name": "product_choice_model.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}