topic_modelling_lda/topic modeling with lda.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NB: This notebook is created based on [this one](https://github.com/sanketg10/nlp-portfolio/blob/master/2.2-topic-modeling/Latent_dirichlet_allocation.ipynb). Any edit was done on my part."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('abcnews-date-text.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "The following steps will be performed:\n",
    "- Tokenization\n",
    "- Stopwords removal\n",
    "- Lemmatization\n",
    "- Stemming\n",
    "'''\n",
    "\n",
    "import gensim\n",
    "from gensim.utils import simple_preprocess\n",
    "from gensim.parsing.preprocessing import STOPWORDS as sw\n",
    "import nltk\n",
    "from nltk.stem import WordNetLemmatizer, SnowballStemmer\n",
    "from nltk.stem.porter import *\n",
    "import numpy as np\n",
    "\n",
    "np.random.seed(12)\n",
    "nltk.download('wordnet')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "stemmer = SnowballStemmer('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lemmatize_stem(text):\n",
    "    return stemmer.stem(WordNetLemmatizer().lemmatize(text, pos='v'))\n",
    "\n",
    "def preprocess(text):\n",
    "    result = []\n",
    "    for token in simple_preprocess(text):\n",
    "        if token not in sw:\n",
    "            result.append(lemmatize_stem(token))\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessed_docs = df['headline_text'][:500000].apply(preprocess)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BoWs on the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Gensim has a tool that can be used to create a dictionary of indexed words\n",
    "with their number of occurrences\n",
    "'''\n",
    "\n",
    "dictionary = gensim.corpora.Dictionary(preprocessed_docs)\n",
    "\n",
    "'''\n",
    "Remove extremes (i.e. very common/very rare words)\n",
    "\n",
    "- words appearing less than 15 times\n",
    "- words appearing in more than 10% of all documents\n",
    "\n",
    "After this, keep the top k=100.000 words \n",
    "'''\n",
    "\n",
    "dictionary.filter_extremes(no_below=15, no_above=0.1, keep_n=100000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Create a BoW model for each document\n",
    "'''\n",
    "\n",
    "bow_corpus = [dictionary.doc2bow(doc) for doc in preprocessed_docs]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TF-IDF on the documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim import corpora, models\n",
    "\n",
    "tfidf = models.TfidfModel(bow_corpus)\n",
    "corpus_tfidf = tfidf[bow_corpus]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running basic LDA on BoW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "lda_model = gensim.models.LdaModel(bow_corpus,\n",
    "                                  num_topics = 10,\n",
    "                                  id2word = dictionary,\n",
    "                                  passes = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic 0: 0.039*\"say\" + 0.023*\"abc\" + 0.015*\"report\" + 0.013*\"bodi\" + 0.013*\"cost\" + 0.012*\"rule\" + 0.011*\"target\" + 0.011*\"oil\" + 0.011*\"work\" + 0.010*\"fin\"\n",
      "\n",
      "\n",
      "Topic 1: 0.035*\"kill\" + 0.033*\"water\" + 0.029*\"attack\" + 0.027*\"home\" + 0.018*\"qld\" + 0.015*\"need\" + 0.014*\"fall\" + 0.013*\"price\" + 0.012*\"bushfir\" + 0.012*\"tiger\"\n",
      "\n",
      "\n",
      "Topic 2: 0.021*\"school\" + 0.017*\"fear\" + 0.016*\"market\" + 0.016*\"road\" + 0.014*\"driver\" + 0.013*\"law\" + 0.013*\"death\" + 0.012*\"blaze\" + 0.012*\"storm\" + 0.011*\"rise\"\n",
      "\n",
      "\n",
      "Topic 3: 0.022*\"day\" + 0.020*\"case\" + 0.019*\"worker\" + 0.019*\"pay\" + 0.017*\"test\" + 0.016*\"china\" + 0.015*\"end\" + 0.015*\"lose\" + 0.012*\"brisban\" + 0.012*\"dead\"\n",
      "\n",
      "\n",
      "Topic 4: 0.032*\"crash\" + 0.024*\"talk\" + 0.022*\"miss\" + 0.017*\"hospit\" + 0.014*\"polic\" + 0.014*\"victim\" + 0.014*\"car\" + 0.013*\"north\" + 0.013*\"rudd\" + 0.013*\"die\"\n",
      "\n",
      "\n",
      "Topic 5: 0.036*\"council\" + 0.018*\"plan\" + 0.015*\"servic\" + 0.015*\"busi\" + 0.015*\"act\" + 0.014*\"take\" + 0.013*\"time\" + 0.013*\"cut\" + 0.011*\"new\" + 0.011*\"budget\"\n",
      "\n",
      "\n",
      "Topic 6: 0.029*\"new\" + 0.022*\"job\" + 0.014*\"power\" + 0.014*\"set\" + 0.014*\"get\" + 0.013*\"open\" + 0.013*\"world\" + 0.012*\"want\" + 0.012*\"centr\" + 0.011*\"cup\"\n",
      "\n",
      "\n",
      "Topic 7: 0.054*\"man\" + 0.039*\"charg\" + 0.029*\"court\" + 0.025*\"face\" + 0.023*\"murder\" + 0.021*\"accus\" + 0.021*\"jail\" + 0.017*\"woman\" + 0.016*\"polic\" + 0.014*\"teen\"\n",
      "\n",
      "\n",
      "Topic 8: 0.060*\"interview\" + 0.039*\"polic\" + 0.021*\"hous\" + 0.020*\"chang\" + 0.016*\"sydney\" + 0.015*\"sex\" + 0.015*\"probe\" + 0.015*\"seek\" + 0.014*\"fund\" + 0.014*\"help\"\n",
      "\n",
      "\n",
      "Topic 9: 0.025*\"flu\" + 0.020*\"swine\" + 0.019*\"australia\" + 0.019*\"govt\" + 0.017*\"fight\" + 0.016*\"urg\" + 0.016*\"australian\" + 0.014*\"green\" + 0.014*\"defend\" + 0.012*\"leav\"\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for idx, topic in lda_model.print_topics(-1):\n",
    "    print(\"Topic {}: {}\".format(idx, topic))\n",
    "    print(\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running LDA using tf-df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "lda_model_tfidf = gensim.models.LdaModel(corpus_tfidf,\n",
    "                                        num_topics = 10,\n",
    "                                        id2word = dictionary,\n",
    "                                        passes=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic 0:0.014*\"woman\" + 0.014*\"rise\" + 0.012*\"storm\" + 0.010*\"guilti\" + 0.010*\"tiger\" + 0.010*\"price\" + 0.008*\"rat\" + 0.008*\"perth\" + 0.008*\"rate\" + 0.008*\"boy\"\n",
      "\n",
      "\n",
      "Topic 1:0.011*\"chang\" + 0.009*\"climat\" + 0.009*\"student\" + 0.009*\"asylum\" + 0.008*\"reveal\" + 0.008*\"river\" + 0.007*\"delay\" + 0.007*\"afl\" + 0.007*\"murray\" + 0.007*\"stimulus\"\n",
      "\n",
      "\n",
      "Topic 2:0.016*\"market\" + 0.012*\"gold\" + 0.010*\"coast\" + 0.010*\"turnbul\" + 0.010*\"budget\" + 0.010*\"rescu\" + 0.009*\"sign\" + 0.009*\"australian\" + 0.009*\"babi\" + 0.008*\"share\"\n",
      "\n",
      "\n",
      "Topic 3:0.021*\"man\" + 0.019*\"polic\" + 0.017*\"charg\" + 0.015*\"murder\" + 0.014*\"court\" + 0.011*\"death\" + 0.010*\"shoot\" + 0.009*\"accus\" + 0.009*\"sex\" + 0.009*\"bushfir\"\n",
      "\n",
      "\n",
      "Topic 4:0.014*\"job\" + 0.011*\"health\" + 0.009*\"school\" + 0.008*\"flood\" + 0.008*\"worker\" + 0.008*\"fund\" + 0.008*\"fear\" + 0.008*\"boost\" + 0.007*\"bodi\" + 0.007*\"teacher\"\n",
      "\n",
      "\n",
      "Topic 5:0.062*\"interview\" + 0.021*\"flu\" + 0.020*\"swine\" + 0.012*\"power\" + 0.010*\"obama\" + 0.007*\"station\" + 0.007*\"tour\" + 0.007*\"adelaid\" + 0.007*\"darwin\" + 0.007*\"test\"\n",
      "\n",
      "\n",
      "Topic 6:0.016*\"dead\" + 0.011*\"kill\" + 0.010*\"blaze\" + 0.010*\"blast\" + 0.009*\"alleg\" + 0.009*\"bus\" + 0.009*\"battl\" + 0.009*\"afghan\" + 0.008*\"go\" + 0.008*\"bomb\"\n",
      "\n",
      "\n",
      "Topic 7:0.017*\"car\" + 0.014*\"crash\" + 0.013*\"final\" + 0.011*\"die\" + 0.011*\"threat\" + 0.010*\"yo\" + 0.010*\"govern\" + 0.007*\"world\" + 0.007*\"develop\" + 0.007*\"steal\"\n",
      "\n",
      "\n",
      "Topic 8:0.020*\"abc\" + 0.012*\"child\" + 0.010*\"run\" + 0.009*\"home\" + 0.009*\"bail\" + 0.009*\"arrest\" + 0.008*\"futur\" + 0.007*\"damag\" + 0.007*\"grant\" + 0.007*\"life\"\n",
      "\n",
      "\n",
      "Topic 9:0.019*\"say\" + 0.015*\"teen\" + 0.012*\"rudd\" + 0.010*\"speak\" + 0.010*\"busi\" + 0.009*\"drive\" + 0.009*\"premier\" + 0.008*\"drink\" + 0.008*\"demand\" + 0.007*\"crisi\"\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for idx, topic in lda_model_tfidf.print_topics(-1):\n",
    "    print(\"Topic {}:{}\".format(idx, topic))\n",
    "    print(\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Document classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Document: adelaide break port hearts\n",
      "\n",
      "Score: 0.4200214445590973\t \n",
      "Topic: 0.036*\"council\" + 0.018*\"plan\" + 0.015*\"servic\" + 0.015*\"busi\" + 0.015*\"act\" + 0.014*\"take\" + 0.013*\"time\" + 0.013*\"cut\" + 0.011*\"new\" + 0.011*\"budget\"\n",
      "\n",
      "Score: 0.21999533474445343\t \n",
      "Topic: 0.029*\"new\" + 0.022*\"job\" + 0.014*\"power\" + 0.014*\"set\" + 0.014*\"get\" + 0.013*\"open\" + 0.013*\"world\" + 0.012*\"want\" + 0.012*\"centr\" + 0.011*\"cup\"\n",
      "\n",
      "Score: 0.21996290981769562\t \n",
      "Topic: 0.021*\"school\" + 0.017*\"fear\" + 0.016*\"market\" + 0.016*\"road\" + 0.014*\"driver\" + 0.013*\"law\" + 0.013*\"death\" + 0.012*\"blaze\" + 0.012*\"storm\" + 0.011*\"rise\"\n",
      "\n",
      "Score: 0.02000472880899906\t \n",
      "Topic: 0.022*\"day\" + 0.020*\"case\" + 0.019*\"worker\" + 0.019*\"pay\" + 0.017*\"test\" + 0.016*\"china\" + 0.015*\"end\" + 0.015*\"lose\" + 0.012*\"brisban\" + 0.012*\"dead\"\n",
      "\n",
      "Score: 0.020004095509648323\t \n",
      "Topic: 0.032*\"crash\" + 0.024*\"talk\" + 0.022*\"miss\" + 0.017*\"hospit\" + 0.014*\"polic\" + 0.014*\"victim\" + 0.014*\"car\" + 0.013*\"north\" + 0.013*\"rudd\" + 0.013*\"die\"\n",
      "\n",
      "Score: 0.02000265009701252\t \n",
      "Topic: 0.039*\"say\" + 0.023*\"abc\" + 0.015*\"report\" + 0.013*\"bodi\" + 0.013*\"cost\" + 0.012*\"rule\" + 0.011*\"target\" + 0.011*\"oil\" + 0.011*\"work\" + 0.010*\"fin\"\n",
      "\n",
      "Score: 0.02000255323946476\t \n",
      "Topic: 0.025*\"flu\" + 0.020*\"swine\" + 0.019*\"australia\" + 0.019*\"govt\" + 0.017*\"fight\" + 0.016*\"urg\" + 0.016*\"australian\" + 0.014*\"green\" + 0.014*\"defend\" + 0.012*\"leav\"\n",
      "\n",
      "Score: 0.020002109929919243\t \n",
      "Topic: 0.035*\"kill\" + 0.033*\"water\" + 0.029*\"attack\" + 0.027*\"home\" + 0.018*\"qld\" + 0.015*\"need\" + 0.014*\"fall\" + 0.013*\"price\" + 0.012*\"bushfir\" + 0.012*\"tiger\"\n",
      "\n",
      "Score: 0.020002109929919243\t \n",
      "Topic: 0.054*\"man\" + 0.039*\"charg\" + 0.029*\"court\" + 0.025*\"face\" + 0.023*\"murder\" + 0.021*\"accus\" + 0.021*\"jail\" + 0.017*\"woman\" + 0.016*\"polic\" + 0.014*\"teen\"\n",
      "\n",
      "Score: 0.020002109929919243\t \n",
      "Topic: 0.060*\"interview\" + 0.039*\"polic\" + 0.021*\"hous\" + 0.020*\"chang\" + 0.016*\"sydney\" + 0.015*\"sex\" + 0.015*\"probe\" + 0.015*\"seek\" + 0.014*\"fund\" + 0.014*\"help\"\n"
     ]
    }
   ],
   "source": [
    "document_num = 700\n",
    "\n",
    "#random test document\n",
    "\n",
    "print(\"Document: {}\".format(df['headline_text'][document_num]))\n",
    "for index, score in sorted(lda_model_tfidf[bow_corpus[document_num]], key=lambda tup: -1*tup[1]):\n",
    "    print(\"\\nScore: {}\\t \\nTopic: {}\".format(score, lda_model.print_topic(index, 10)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification of unseen document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score: 0.3666568398475647\t Topic: 0.016*\"market\" + 0.012*\"gold\" + 0.010*\"coast\" + 0.010*\"turnbul\" + 0.010*\"budget\"\n",
      "score: 0.36662834882736206\t Topic: 0.062*\"interview\" + 0.021*\"flu\" + 0.020*\"swine\" + 0.012*\"power\" + 0.010*\"obama\"\n",
      "score: 0.033339351415634155\t Topic: 0.014*\"woman\" + 0.014*\"rise\" + 0.012*\"storm\" + 0.010*\"guilti\" + 0.010*\"tiger\"\n",
      "score: 0.033339351415634155\t Topic: 0.011*\"chang\" + 0.009*\"climat\" + 0.009*\"student\" + 0.009*\"asylum\" + 0.008*\"reveal\"\n",
      "score: 0.033339351415634155\t Topic: 0.021*\"man\" + 0.019*\"polic\" + 0.017*\"charg\" + 0.015*\"murder\" + 0.014*\"court\"\n",
      "score: 0.033339351415634155\t Topic: 0.014*\"job\" + 0.011*\"health\" + 0.009*\"school\" + 0.008*\"flood\" + 0.008*\"worker\"\n",
      "score: 0.033339351415634155\t Topic: 0.016*\"dead\" + 0.011*\"kill\" + 0.010*\"blaze\" + 0.010*\"blast\" + 0.009*\"alleg\"\n",
      "score: 0.033339351415634155\t Topic: 0.017*\"car\" + 0.014*\"crash\" + 0.013*\"final\" + 0.011*\"die\" + 0.011*\"threat\"\n",
      "score: 0.033339351415634155\t Topic: 0.020*\"abc\" + 0.012*\"child\" + 0.010*\"run\" + 0.009*\"home\" + 0.009*\"bail\"\n",
      "score: 0.033339351415634155\t Topic: 0.019*\"say\" + 0.015*\"teen\" + 0.012*\"rudd\" + 0.010*\"speak\" + 0.010*\"busi\"\n"
     ]
    }
   ],
   "source": [
    "unseen_document = \"I love swimming.\"\n",
    "\n",
    "bow_vector = dictionary.doc2bow(preprocess(unseen_document))\n",
    "\n",
    "for idx, score in sorted(lda_model[bow_vector], key=lambda tup: -1*tup[1]):\n",
    "    print(\"score: {}\\t Topic: {}\".format(score, lda_model_tfidf.print_topic(idx, 5)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}