Claims Topic Modeling 10.3.21.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CorEx Topic Modeling to detect recurrent themes in the claims text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import sys\n",
    "import spacy\n",
    "# !{sys.executable} -m spacy download en\n",
    "\n",
    "import gensim\n",
    "from corextopic import corextopic as ct\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction import text\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create claims dataset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# true claims\n",
    "\n",
    "real_claims1 = pd.read_csv(\"https://raw.githubusercontent.com/cuilimeng/CoAID/master/05-01-2020/ClaimRealCOVID-19.csv\")\n",
    "real_claims2 = pd.read_csv(\"https://raw.githubusercontent.com/cuilimeng/CoAID/master/07-01-2020/ClaimRealCOVID-19.csv\")\n",
    "real_claims3 = pd.read_csv(\"https://raw.githubusercontent.com/cuilimeng/CoAID/master/09-01-2020/ClaimRealCOVID-19.csv\")\n",
    "real_claims4 = pd.read_csv(\"https://raw.githubusercontent.com/cuilimeng/CoAID/master/11-01-2020/ClaimRealCOVID-19.csv\")\n",
    "\n",
    "real_claims = pd.concat([real_claims1, real_claims2, real_claims3, real_claims4])\n",
    "\n",
    "real_claims['eval'] = 'real'\n",
    "\n",
    "# fake claims- none for 9/1 and 11/1 additions\n",
    "\n",
    "fake_claims1 = pd.read_csv(\"https://raw.githubusercontent.com/cuilimeng/CoAID/master/05-01-2020/ClaimFakeCOVID-19.csv\")\n",
    "fake_claims2 = pd.read_csv(\"https://raw.githubusercontent.com/cuilimeng/CoAID/master/07-01-2020/ClaimFakeCOVID-19.csv\")\n",
    "\n",
    "fake_claims = pd.concat([fake_claims1, fake_claims2])\n",
    "\n",
    "fake_claims['eval'] = 'fake'\n",
    "\n",
    "claims = pd.concat([real_claims, fake_claims], axis = 0)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text Pre-Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['how large meeting event need order mass gathering', 'recommend international mass gathering cancel']\n"
     ]
    }
   ],
   "source": [
    "# Convert review text to list\n",
    "data = claims.title.values.tolist()\n",
    "\n",
    "\n",
    "# Tokenize by sentence- each review becomes a list of words\n",
    "def sent_to_words(sentences):\n",
    "    for sentence in sentences:\n",
    "        yield(gensim.utils.simple_preprocess(str(sentence), deacc = True))  # deacc=True removes punctuations\n",
    "\n",
    "data_words = list(sent_to_words(data))\n",
    "\n",
    "\n",
    "# Only keep certain parts of speech\n",
    "def lemmatization(texts, allowed_postags = ['NOUN', 'ADJ', 'VERB', 'ADV']): #'NOUN', 'ADJ', 'VERB', 'ADV'\n",
    "    texts_out = []\n",
    "    for sent in texts:\n",
    "        doc = nlp(' '.join(sent)) \n",
    "        texts_out.append(' '.join([token.lemma_ if token.lemma_ not in ['-PRON-'] else '' for token in doc if token.pos_ in allowed_postags]))\n",
    "    return texts_out\n",
    "\n",
    "\n",
    "nlp = spacy.load('en', disable = ['parser', 'ner'])\n",
    "\n",
    "data_lemmatized = lemmatization(data_words, allowed_postags = ['NOUN', 'VERB', 'ADJ', 'ADV']) #select noun and verb\n",
    "\n",
    "print(data_lemmatized[:2])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vectorizing the text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3682\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_extraction/text.py:383: UserWarning: Your stop_words may be inconsistent with your preprocessing. Tokenizing the stop words generated tokens ['19'] not in stop_words.\n",
      "  warnings.warn('Your stop_words may be inconsistent with '\n"
     ]
    }
   ],
   "source": [
    "# Vectorizing\n",
    "my_words = set(['covid', 'covid-19', 'coronavirus', 'virus', 'patient', 'sick'])\n",
    "\n",
    "my_stop_words = text.ENGLISH_STOP_WORDS.union(my_words)\n",
    "\n",
    "vectorizer = TfidfVectorizer(\n",
    "    analyzer = 'word', \n",
    "    max_df = .95, # ignore terms that appear in more than 95% of the documents\n",
    "    #min_df = 0.05, # ignore terms that appear in less than 5% of the documents\"\n",
    "    max_features = None, # default\n",
    "    ngram_range = (2, 4),\n",
    "    #norm = None, # default = l2\n",
    "    binary = False, # default = False\n",
    "    use_idf = True, # make sure to set to True so that it's actually used\n",
    "    sublinear_tf = False, # default\n",
    "    stop_words = set(my_stop_words) # edited as above\n",
    ")\n",
    "\n",
    "\n",
    "# Fit on the text of the claims\n",
    "data_lemmatized\n",
    "\n",
    "vectorizer = vectorizer.fit(data_lemmatized)\n",
    "tfidf = vectorizer.transform(data_lemmatized)\n",
    "vocab = vectorizer.get_feature_names()\n",
    "\n",
    "print(len(vocab))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CorEx Topic Modeling- Unseeded Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic #1: hand dryer, member international health regulation, member international, member international health, mask general\n",
      "Topic #2: family planning, contraception family, contraception family planning, drug license treatment prevention, license treatment prevention\n",
      "Topic #3: self care, position use chloroquine context, position use chloroquine, chloroquine context, chloroquine context response\n",
      "Topic #4: develop severe, risk develop severe, risk develop, contraceptive method, change guidance respect malaria\n",
      "Topic #5: live surface food, live surface, surface food, surface food packaging, live surface food packaging\n",
      "Topic #6: drink alcohol protect, drink alcohol protect dangerous, alcohol protect dangerous, alcohol protect, spread coin banknote\n",
      "Topic #7: malaria affect country, affect country, malaria affect, good household disinfectant surface, good household disinfectant\n",
      "Topic #8: wash fruit vegetable, wash fruit, prevent treat, vegetable time, wash fruit vegetable time\n",
      "Topic #9: drug treatment, vaccine drug, vaccine drug treatment, incubation period, touch hold newborn\n",
      "Topic #10: right duty, affect area, spread malaria affect area, spread malaria affect, spread malaria\n",
      "Topic #11: address ventilation, ventilation context, address ventilation context, vaccine pneumonia, vaccine pneumonia protect\n",
      "Topic #12: preventive therapy, preventive therapy maintain, recommend preventive, recommend preventive therapy, recommend preventive therapy maintain\n",
      "Topic #13: pregnant woman, woman test, pregnant woman test, woman high, woman high risk\n",
      "Topic #14: mask protect, thermal scanner, wear mask protect, thermal scanner detect, scanner detect\n",
      "Topic #15: accommodation facility, child transmission, role child, role child transmission, traveler accommodation\n"
     ]
    }
   ],
   "source": [
    "# Initiate Corex Model \n",
    "# 15 topics\n",
    "\n",
    "model_15 = ct.Corex(n_hidden = 15, seed = 123, n_jobs = -1) \n",
    "model_15 = model_15.fit(tfidf, words = vocab)\n",
    "\n",
    "# Print the top 5 words of each topic- no anchors\n",
    "for i, topic_ngrams in enumerate(model_15.get_topics(n_words = 5)):\n",
    "    topic_ngrams = [ngram[0] for ngram in topic_ngrams if ngram[1] > 0]\n",
    "    print('Topic #{}: {}'.format(i+1, ', '.join(topic_ngrams)))\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic #1: address ventilation, address ventilation context, ventilation context, food grocery delivery, food grocery delivery safe\n",
      "Topic #2: good household disinfectant, household disinfectant surface, good household disinfectant surface, good household, disinfectant surface\n",
      "Topic #3: develop severe, risk develop, risk develop severe, severe symptom, concerned spread\n",
      "Topic #4: safely time, grocery shop safely time, shop safely, shop safely time, grocery shop safely\n",
      "Topic #5: chloroquine context response, use chloroquine, chloroquine context, use chloroquine context, use chloroquine context response\n",
      "Topic #6: right duty, medical mask, old people, surface food packaging, food packaging\n",
      "Topic #7: preventive therapy maintain, preventive therapy, therapy maintain, recommend preventive, recommend preventive therapy maintain\n",
      "Topic #8: incubation period, mask general public, general public, mask general, long incubation period\n",
      "Topic #9: affect country, malaria affect country, malaria affect, country report case, country report\n",
      "Topic #10: help prevent, eat garlic, eat garlic help, eat garlic help prevent, garlic help\n",
      "Topic #11: transportation school, people room, room previous, risk transportation school, people room previous\n",
      "Topic #12: prevent treat, treat new, prevent treat new, vaccine drug treatment, drug treatment\n",
      "Topic #13: alcohol protect, alcohol protect dangerous, drink alcohol protect dangerous, drink alcohol protect, drink alcohol\n",
      "Topic #14: dose use recovery, recovery trial, use recovery, use recovery trial, dose use recovery trial\n",
      "Topic #15: fruit vegetable time, vegetable time, wash fruit vegetable time, wash fruit, wash fruit vegetable\n",
      "Topic #16: pregnant woman, pregnant woman test, woman test, pregnant woman high, pregnant woman high risk\n",
      "Topic #17: coin banknote, spread coin, spread coin banknote, child transmission, role child\n",
      "Topic #18: thermal scanner, scanner detect, thermal scanner detect, scanner diagnose, thermal scanner diagnose\n",
      "Topic #19: mineral supplement cure, supplement cure, vitamin mineral supplement cure, mineral supplement, vitamin mineral\n",
      "Topic #20: pneumonia protect, vaccine pneumonia, vaccine pneumonia protect\n"
     ]
    }
   ],
   "source": [
    "# 20 topics\n",
    "\n",
    "model_20 = ct.Corex(n_hidden = 20, seed = 123, n_jobs = -1) #20 is good\n",
    "model_20 = model_20.fit(tfidf, words = vocab)\n",
    "\n",
    "# Print the top 5 words of each topic\n",
    "for i, topic_ngrams in enumerate(model_20.get_topics(n_words = 5)):\n",
    "    topic_ngrams = [ngram[0] for ngram in topic_ngrams if ngram[1] > 0]\n",
    "    print('Topic #{}: {}'.format(i+1, ', '.join(topic_ngrams)))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic #1: assess risk, tuberculosis spread way, tuberculosis spread, spread way, medical face mask\n",
      "Topic #2: likelihood shoe spread low, likelihood shoe spread, likelihood shoe, shoe spread, shoe spread low\n",
      "Topic #3: thermal scanner detect, scanner detect, medical mask, antiretroviral use, thermal scanner\n",
      "Topic #4: contraceptive method, respect malaria diagnosis treatment, respect malaria diagnosis, guidance respect, guidance respect malaria\n",
      "Topic #5: wash fruit vegetable, wash fruit, fruit vegetable, fruit vegetable time, vegetable time\n",
      "Topic #6: risk develop severe, develop severe, risk develop, vaccine drug treatment, vaccine drug\n",
      "Topic #7: newborn baby, international health regulation, health regulation, alcohol protect, alcohol protect dangerous\n",
      "Topic #8: pregnant woman, woman test, pregnant woman test, affect country report, affect country report case\n",
      "Topic #9: help prevent, eat garlic, eat garlic help, eat garlic help prevent, garlic help\n",
      "Topic #10: preventive therapy maintain, recommend preventive, recommend preventive therapy, recommend preventive therapy maintain, preventive therapy\n",
      "Topic #11: incubation period, antibiotic effective, antibiotic effective prevent, antibiotic effective prevent treat, effective prevent treat\n",
      "Topic #12: additional special, additional special measure, additional special measure need, special measure need context, special measure need\n",
      "Topic #13: affect country context, support malaria affect country, support malaria affect, country context, support malaria\n",
      "Topic #14: disinfectant surface, good household, good household disinfectant, good household disinfectant surface, household disinfectant\n",
      "Topic #15: address ventilation, ventilation context, address ventilation context, vaccine pneumonia protect, vaccine pneumonia\n",
      "Topic #16: delivery safe, food grocery, food grocery delivery, food grocery delivery safe, grocery delivery\n",
      "Topic #17: mask protect, wear mask protect, wear mask, child wear mask, child wear\n",
      "Topic #18: right duty, scanner diagnose, thermal scanner diagnose, duty employer, duty worker\n",
      "Topic #19: catch person symptom, person symptom, safe exercise, stay safe exercise, stay safe\n",
      "Topic #20: grocery deliver, safe grocery deliver, symptom infection\n",
      "Topic #21: define contact\n"
     ]
    }
   ],
   "source": [
    "# 21 topics\n",
    "\n",
    "model_21 = ct.Corex(n_hidden = 21, seed = 123, n_jobs = -1) \n",
    "model_21 = model_21.fit(tfidf, words = vocab)\n",
    "\n",
    "# Print the top 5 words of each topic\n",
    "for i, topic_ngrams in enumerate(model_21.get_topics(n_words = 5)):\n",
    "    topic_ngrams = [ngram[0] for ngram in topic_ngrams if ngram[1] > 0]\n",
    "    print('Topic #{}: {}'.format(i+1, ', '.join(topic_ngrams)))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic #1: safely time, grocery shop safely time, grocery shop safely, grocery shop, shop safely time\n",
      "Topic #2: health authority, assess risk, kill new, use pandemic, measure people\n",
      "Topic #3: household disinfectant surface, household disinfectant, good household disinfectant, good household disinfectant surface, disinfectant surface\n",
      "Topic #4: live surface food, live surface, additional special, special measure need context, special measure need\n",
      "Topic #5: support malaria affect country, affect country context, support malaria affect, country context, malaria affect country context\n",
      "Topic #6: prevent cure, survive surface, respect malaria, change guidance respect malaria, change guidance respect\n",
      "Topic #7: pregnant woman, woman test, pregnant woman test, vaccine pneumonia, vaccine pneumonia protect\n",
      "Topic #8: pregnancy childbirth, available pregnancy, care available, care available pregnancy, available pregnancy childbirth\n",
      "Topic #9: preventive therapy, preventive therapy maintain, recommend preventive therapy maintain, therapy maintain, recommend preventive\n",
      "Topic #10: international health regulation, health regulation, right duty, affect area, spread malaria affect\n",
      "Topic #11: precaution consumer, precaution consumer grocery store, consumer grocery, consumer grocery store, precaution consumer grocery\n",
      "Topic #12: hold newborn, hold newborn baby, touch hold newborn baby, touch hold newborn, touch hold\n",
      "Topic #13: affect country report, affect country report case, country report, country report case, report case\n",
      "Topic #14: develop severe, risk develop severe, risk develop, vaccine trial, volunteer vaccine\n",
      "Topic #15: alcohol protect, alcohol protect dangerous, drink alcohol protect, drink alcohol protect dangerous, drink alcohol\n",
      "Topic #16: fruit vegetable time, vegetable time, wash fruit vegetable time, wash fruit, wash fruit vegetable\n",
      "Topic #17: vaccine drug treatment, drug treatment, vaccine drug, incubation period, child care\n",
      "Topic #18: address ventilation, address ventilation context, ventilation context, coin banknote, spread coin\n",
      "Topic #19: contact trace work, trace work, recommend use dexamethasone, use dexamethasone, safe exercise\n",
      "Topic #20: cause know, monitor opening, monitor opening school, opening school, source cause\n",
      "Topic #21: mask protect, wear mask protect, face mask protect, face mask, wear mask\n",
      "Topic #22: dexamethasone work, catch fece, catch fece disease, fece disease\n"
     ]
    }
   ],
   "source": [
    "# 22 topics\n",
    "\n",
    "model_22 = ct.Corex(n_hidden = 22, seed = 123, n_jobs = -1) \n",
    "model_22 = model_22.fit(tfidf, words = vocab)\n",
    "\n",
    "# Print the top 5 words of each topic\n",
    "for i, topic_ngrams in enumerate(model_22.get_topics(n_words = 5)):\n",
    "    topic_ngrams = [ngram[0] for ngram in topic_ngrams if ngram[1] > 0]\n",
    "    print('Topic #{}: {}'.format(i+1, ', '.join(topic_ngrams)))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic #1: risk transportation, risk transportation school, transportation school, smokeless tobacco, sport tournament\n",
      "Topic #2: contraceptive method, grocery delivery, shop safely, shop safely time, delivery safe\n",
      "Topic #3: risk develop, risk develop severe, develop severe, likelihood shoe spread, spread low\n",
      "Topic #4: malaria affect, malaria affect country, affect country, malaria affect country report, support malaria\n",
      "Topic #5: recommend preventive, recommend preventive therapy, therapy maintain, preventive therapy, preventive therapy maintain\n",
      "Topic #6: chloroquine context, position use chloroquine, position use chloroquine context, context response, chloroquine context response\n",
      "Topic #7: guidance respect malaria diagnosis, guidance respect, guidance respect malaria, change guidance, change guidance respect\n",
      "Topic #8: prevent treat, treat new, prevent treat new, antibiotic effective, antibiotic effective prevent\n",
      "Topic #9: household disinfectant, good household disinfectant surface, good household disinfectant, good household, household disinfectant surface\n",
      "Topic #10: mask general public, mask general, general public, swimming pool, catch swimming pool\n",
      "Topic #11: vegetable time, fruit vegetable time, wash fruit vegetable time, wash fruit vegetable, wash fruit\n",
      "Topic #12: alcohol protect, alcohol protect dangerous, drink alcohol protect, drink alcohol protect dangerous, spread coin banknote\n",
      "Topic #13: pregnant woman high, pregnant woman high risk, woman high, woman high risk, mineral supplement cure\n",
      "Topic #14: mask protect, health regulation, international health regulation, wear mask protect, international health\n",
      "Topic #15: vaccine pneumonia, vaccine pneumonia protect, pneumonia protect, available world, departure arrival\n",
      "Topic #16: contact trace work, trace work, antiretroviral use treat, grocery deliver, safe grocery deliver\n",
      "Topic #17: drug treatment, vaccine drug, vaccine drug treatment\n",
      "Topic #18: mask exercise, people wear, people wear mask, people wear mask exercise, wear mask exercise\n",
      "Topic #19: leverage disease, service leverage, service leverage disease, address ventilation, address ventilation context\n",
      "Topic #20: scanner detect, thermal scanner detect, thermal scanner, result mean, ncov spread\n",
      "Topic #21: scanner diagnose, thermal scanner diagnose, people room, people room previous, room previous\n",
      "Topic #22: pregnant woman test, woman test, cause transmit air, transmit air, cause transmit\n",
      "Topic #23: confirm ncov infection, ncov infection, suspect confirm ncov infection, confirm ncov, suspect confirm ncov\n"
     ]
    }
   ],
   "source": [
    "# 23 topics\n",
    "\n",
    "model_23 = ct.Corex(n_hidden = 23, seed = 123, n_jobs = -1)\n",
    "model_23 = model_23.fit(tfidf, words = vocab)\n",
    "\n",
    "# Print the top 5 words of each topic\n",
    "for i, topic_ngrams in enumerate(model_23.get_topics(n_words = 5)):\n",
    "    topic_ngrams = [ngram[0] for ngram in topic_ngrams if ngram[1] > 0]\n",
    "    print('Topic #{}: {}'.format(i+1, ', '.join(topic_ngrams)))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic #1: garlic help, garlic help prevent, eat garlic, eat garlic help prevent, eat garlic help\n",
      "Topic #2: protect dangerous, alcohol protect dangerous, alcohol protect, drink alcohol protect, drink alcohol protect dangerous\n",
      "Topic #3: key consideration, workplace risk assessment, risk assessment, workplace risk, country work\n",
      "Topic #4: prevent treat, treat new, prevent treat new, antibiotic effective prevent treat, antibiotic effective\n",
      "Topic #5: position use, position use chloroquine, use chloroquine context response, use chloroquine context, use chloroquine\n",
      "Topic #6: wash fruit vegetable time, fruit vegetable time, vegetable time, good household disinfectant, disinfectant surface\n",
      "Topic #7: particularly concerned spread malaria, particularly concerned, spread malaria, spread malaria affect, spread malaria affect area\n",
      "Topic #8: risk develop, develop severe, risk develop severe, severe symptom, vitamin mineral supplement cure\n",
      "Topic #9: likelihood shoe, likelihood shoe spread low, spread low, likelihood shoe spread, shoe spread low\n",
      "Topic #10: woman test, pregnant woman test, pregnant woman, use pandemic, method use\n",
      "Topic #11: stay safe, hotel accommodation, hotel accommodation establishment, child care, child care home\n",
      "Topic #12: malaria affect country report, affect country report, affect country report case, country report, country report case\n",
      "Topic #13: right duty, vaccine pneumonia protect, vaccine pneumonia, pneumonia protect, available pregnancy\n",
      "Topic #14: preventive therapy, preventive therapy maintain, recommend preventive, recommend preventive therapy, recommend preventive therapy maintain\n",
      "Topic #15: health regulation, international health regulation, international health, member international, member international health\n",
      "Topic #16: mask protect, wear mask protect, incubation period, face mask protect, period child\n",
      "Topic #17: thermal scanner, thermal scanner detect, scanner detect, probiotic help prevent, probiotic help\n",
      "Topic #18: coin banknote, spread coin, spread coin banknote, grocery deliver, safe grocery deliver\n",
      "Topic #19: contact trace work, trace work, consider datum, consider datum protection, datum protection\n",
      "Topic #20: risk transportation, risk transportation school, transportation school\n",
      "Topic #21: address ventilation, address ventilation context, ventilation context\n",
      "Topic #22: long survive, long survive surface, novel infection, treatment novel, treatment novel infection\n",
      "Topic #23: \n",
      "Topic #24: \n"
     ]
    }
   ],
   "source": [
    "# 24 topics\n",
    "\n",
    "model_24 = ct.Corex(n_hidden = 24, seed = 123, n_jobs = -1) \n",
    "model_24 = model_24.fit(tfidf, words = vocab)\n",
    "\n",
    "# Print the top 5 words of each topic- no anchors\n",
    "for i, topic_ngrams in enumerate(model_24.get_topics(n_words = 5)):\n",
    "    topic_ngrams = [ngram[0] for ngram in topic_ngrams if ngram[1] > 0]\n",
    "    print('Topic #{}: {}'.format(i+1, ', '.join(topic_ngrams)))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic #1: spread coin banknote, coin banknote, spread coin, food business, health condition\n",
      "Topic #2: right duty, breastfeed baby directly, baby directly, breastfeed baby, unwell breastfeed\n",
      "Topic #3: healthcare worker, planning information, family planning information, family planning information service, people access contraception family\n",
      "Topic #4: health regulation, international health regulation, woman test, pregnant woman test, international health\n",
      "Topic #5: live surface food, live surface, incubation period, surface food, guidance respect malaria diagnosis\n",
      "Topic #6: contact tracing, context response, position use chloroquine context, use chloroquine context response, use chloroquine context\n",
      "Topic #7: wash fruit, wash fruit vegetable, fruit vegetable time, vegetable time, wash fruit vegetable time\n",
      "Topic #8: household disinfectant surface, good household disinfectant surface, disinfectant surface, household disinfectant, good household disinfectant\n",
      "Topic #9: prevent treat, prevent treat new, treat new, effective prevent treat, antibiotic effective\n",
      "Topic #10: malaria affect country, affect country, malaria affect, preventive therapy, recommend preventive\n",
      "Topic #11: address ventilation, ventilation context, address ventilation context, eat garlic, eat garlic help\n",
      "Topic #12: alcohol protect, alcohol protect dangerous, drink alcohol protect dangerous, drink alcohol protect, drink alcohol\n",
      "Topic #13: ventilation air conditioning use, ventilation air, air conditioning use, conditioning use, ventilation air conditioning\n",
      "Topic #14: hold newborn, hold newborn baby, touch hold newborn baby, touch hold newborn, touch hold\n",
      "Topic #15: pneumonia protect, vaccine pneumonia, vaccine pneumonia protect, dryer kill, hand dryer kill\n",
      "Topic #16: mask protect, wear mask protect, wear mask, child wear mask, child wear\n",
      "Topic #17: recommend use dexamethasone, use dexamethasone, cause transmit air, transmit air, cause transmit\n",
      "Topic #18: drug treatment, vaccine drug, vaccine drug treatment\n",
      "Topic #19: grocery deliver, safe grocery deliver, safe grocery\n",
      "Topic #20: essential travel, mean essential, mean essential travel\n",
      "Topic #21: probiotic help, probiotic help prevent, child care, child care home, child transmission\n",
      "Topic #22: scanner detect, thermal scanner detect, thermal scanner\n",
      "Topic #23: transmit housefly, scanner diagnose, thermal scanner diagnose\n",
      "Topic #24: \n",
      "Topic #25: \n"
     ]
    }
   ],
   "source": [
    "# 25 topics\n",
    "\n",
    "model_25 = ct.Corex(n_hidden = 25, seed = 123, n_jobs = -1) \n",
    "model_25 = model_25.fit(tfidf, words = vocab)\n",
    "\n",
    "# Print the top 5 words of each topic- no anchors\n",
    "for i, topic_ngrams in enumerate(model_25.get_topics(n_words = 5)):\n",
    "    topic_ngrams = [ngram[0] for ngram in topic_ngrams if ngram[1] > 0]\n",
    "    print('Topic #{}: {}'.format(i+1, ', '.join(topic_ngrams)))\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking topic correlation\n",
    "- although the tc is higher as the number of topics increases, the interpretability of the topics suffers. A topic count of 20 seems to offer a good balance between tc and topic interpretability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.0800399088135504\n",
      "3.46576662139439\n",
      "3.3357408892366074\n",
      "3.3674525721472532\n",
      "3.3050751905246285\n",
      "3.420543984571386\n",
      "3.5740708656114055\n"
     ]
    }
   ],
   "source": [
    "# Overall higher tc = better model, produces topics that are most informative about the claims\n",
    "# Search for best number of topics- where tc is the greatest, but also reasonable for interpretation\n",
    "\n",
    "print(model_15.tc)\n",
    "print(model_20.tc)\n",
    "print(model_21.tc)\n",
    "print(model_22.tc)\n",
    "print(model_23.tc)\n",
    "print(model_24.tc)\n",
    "print(model_25.tc)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Topic probabilities for each claim using topic count of 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.003132</td>\n",
       "      <td>0.001405</td>\n",
       "      <td>0.010406</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.004620</td>\n",
       "      <td>0.011083</td>\n",
       "      <td>0.010577</td>\n",
       "      <td>0.00106</td>\n",
       "      <td>0.002529</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.000566</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>0.008207</td>\n",
       "      <td>0.00138</td>\n",
       "      <td>0.001034</td>\n",
       "      <td>0.001823</td>\n",
       "      <td>0.002395</td>\n",
       "      <td>0.002773</td>\n",
       "      <td>0.175692</td>\n",
       "      <td>0.169631</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.003132</td>\n",
       "      <td>0.001405</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.009513</td>\n",
       "      <td>0.004620</td>\n",
       "      <td>0.011083</td>\n",
       "      <td>0.010577</td>\n",
       "      <td>0.00106</td>\n",
       "      <td>0.002529</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.000566</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>0.008207</td>\n",
       "      <td>0.00138</td>\n",
       "      <td>0.000522</td>\n",
       "      <td>0.001823</td>\n",
       "      <td>0.002395</td>\n",
       "      <td>0.002773</td>\n",
       "      <td>0.173437</td>\n",
       "      <td>0.169430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.001405</td>\n",
       "      <td>0.010406</td>\n",
       "      <td>0.009513</td>\n",
       "      <td>0.004620</td>\n",
       "      <td>0.011083</td>\n",
       "      <td>0.010577</td>\n",
       "      <td>0.00106</td>\n",
       "      <td>0.002529</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.000566</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>0.008207</td>\n",
       "      <td>0.00138</td>\n",
       "      <td>0.001034</td>\n",
       "      <td>0.001228</td>\n",
       "      <td>0.002395</td>\n",
       "      <td>0.002773</td>\n",
       "      <td>0.175692</td>\n",
       "      <td>0.170225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.005846</td>\n",
       "      <td>0.001405</td>\n",
       "      <td>0.010406</td>\n",
       "      <td>0.009513</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.011083</td>\n",
       "      <td>0.010577</td>\n",
       "      <td>0.00106</td>\n",
       "      <td>0.002529</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.000566</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>0.008207</td>\n",
       "      <td>0.00138</td>\n",
       "      <td>0.001034</td>\n",
       "      <td>0.001823</td>\n",
       "      <td>0.002395</td>\n",
       "      <td>0.002773</td>\n",
       "      <td>0.175692</td>\n",
       "      <td>0.169681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.001405</td>\n",
       "      <td>0.010406</td>\n",
       "      <td>0.009513</td>\n",
       "      <td>0.006246</td>\n",
       "      <td>0.011083</td>\n",
       "      <td>0.010577</td>\n",
       "      <td>0.00106</td>\n",
       "      <td>0.002529</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.000566</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>0.008207</td>\n",
       "      <td>0.00138</td>\n",
       "      <td>0.000641</td>\n",
       "      <td>0.001823</td>\n",
       "      <td>0.002395</td>\n",
       "      <td>0.002773</td>\n",
       "      <td>0.174114</td>\n",
       "      <td>0.169673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>513</th>\n",
       "      <td>0.003132</td>\n",
       "      <td>0.001405</td>\n",
       "      <td>0.010406</td>\n",
       "      <td>0.009513</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.011083</td>\n",
       "      <td>0.010577</td>\n",
       "      <td>0.00106</td>\n",
       "      <td>0.002529</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.000566</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>0.008207</td>\n",
       "      <td>0.00138</td>\n",
       "      <td>0.001034</td>\n",
       "      <td>0.001823</td>\n",
       "      <td>0.002395</td>\n",
       "      <td>0.002773</td>\n",
       "      <td>0.175692</td>\n",
       "      <td>0.170237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>514</th>\n",
       "      <td>0.003132</td>\n",
       "      <td>0.001405</td>\n",
       "      <td>0.010406</td>\n",
       "      <td>0.009513</td>\n",
       "      <td>0.004620</td>\n",
       "      <td>0.011083</td>\n",
       "      <td>0.010577</td>\n",
       "      <td>0.00106</td>\n",
       "      <td>0.002529</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.000566</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>0.008207</td>\n",
       "      <td>0.00138</td>\n",
       "      <td>0.001034</td>\n",
       "      <td>0.001823</td>\n",
       "      <td>0.002395</td>\n",
       "      <td>0.002773</td>\n",
       "      <td>0.175692</td>\n",
       "      <td>0.170237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>515</th>\n",
       "      <td>0.003132</td>\n",
       "      <td>0.001405</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.009513</td>\n",
       "      <td>0.007108</td>\n",
       "      <td>0.011083</td>\n",
       "      <td>0.010577</td>\n",
       "      <td>0.00106</td>\n",
       "      <td>0.002529</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.000566</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>0.028109</td>\n",
       "      <td>0.00138</td>\n",
       "      <td>0.001034</td>\n",
       "      <td>0.001823</td>\n",
       "      <td>0.002395</td>\n",
       "      <td>0.002773</td>\n",
       "      <td>0.175692</td>\n",
       "      <td>0.170237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>516</th>\n",
       "      <td>0.003132</td>\n",
       "      <td>0.001405</td>\n",
       "      <td>0.020789</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.004620</td>\n",
       "      <td>0.011083</td>\n",
       "      <td>0.010577</td>\n",
       "      <td>0.00106</td>\n",
       "      <td>0.002529</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.000566</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>0.008207</td>\n",
       "      <td>0.00138</td>\n",
       "      <td>0.001034</td>\n",
       "      <td>0.001823</td>\n",
       "      <td>0.002395</td>\n",
       "      <td>0.002773</td>\n",
       "      <td>0.175692</td>\n",
       "      <td>0.170237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>517</th>\n",
       "      <td>0.003132</td>\n",
       "      <td>0.001405</td>\n",
       "      <td>0.010406</td>\n",
       "      <td>0.009513</td>\n",
       "      <td>0.004620</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.010577</td>\n",
       "      <td>0.00106</td>\n",
       "      <td>0.002529</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.000566</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>0.008207</td>\n",
       "      <td>0.00138</td>\n",
       "      <td>0.001034</td>\n",
       "      <td>0.001823</td>\n",
       "      <td>0.002395</td>\n",
       "      <td>0.002773</td>\n",
       "      <td>0.175692</td>\n",
       "      <td>0.170237</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>518 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           0         1         2         3         4         5         6   \\\n",
       "0    0.003132  0.001405  0.010406  0.999999  0.004620  0.011083  0.010577   \n",
       "1    0.003132  0.001405  0.999999  0.009513  0.004620  0.011083  0.010577   \n",
       "2    0.999999  0.001405  0.010406  0.009513  0.004620  0.011083  0.010577   \n",
       "3    0.005846  0.001405  0.010406  0.009513  0.999999  0.011083  0.010577   \n",
       "4    0.999999  0.001405  0.010406  0.009513  0.006246  0.011083  0.010577   \n",
       "..        ...       ...       ...       ...       ...       ...       ...   \n",
       "513  0.003132  0.001405  0.010406  0.009513  0.999999  0.011083  0.010577   \n",
       "514  0.003132  0.001405  0.010406  0.009513  0.004620  0.011083  0.010577   \n",
       "515  0.003132  0.001405  0.999999  0.009513  0.007108  0.011083  0.010577   \n",
       "516  0.003132  0.001405  0.020789  0.999999  0.004620  0.011083  0.010577   \n",
       "517  0.003132  0.001405  0.010406  0.009513  0.004620  0.999999  0.010577   \n",
       "\n",
       "          7         8         9         10        11        12       13  \\\n",
       "0    0.00106  0.002529  0.000719  0.000566  0.001685  0.008207  0.00138   \n",
       "1    0.00106  0.002529  0.000719  0.000566  0.001685  0.008207  0.00138   \n",
       "2    0.00106  0.002529  0.000719  0.000566  0.001685  0.008207  0.00138   \n",
       "3    0.00106  0.002529  0.000719  0.000566  0.001685  0.008207  0.00138   \n",
       "4    0.00106  0.002529  0.000719  0.000566  0.001685  0.008207  0.00138   \n",
       "..       ...       ...       ...       ...       ...       ...      ...   \n",
       "513  0.00106  0.002529  0.000719  0.000566  0.001685  0.008207  0.00138   \n",
       "514  0.00106  0.002529  0.000719  0.000566  0.001685  0.008207  0.00138   \n",
       "515  0.00106  0.002529  0.000719  0.000566  0.001685  0.028109  0.00138   \n",
       "516  0.00106  0.002529  0.000719  0.000566  0.001685  0.008207  0.00138   \n",
       "517  0.00106  0.002529  0.000719  0.000566  0.001685  0.008207  0.00138   \n",
       "\n",
       "           14        15        16        17        18        19  \n",
       "0    0.001034  0.001823  0.002395  0.002773  0.175692  0.169631  \n",
       "1    0.000522  0.001823  0.002395  0.002773  0.173437  0.169430  \n",
       "2    0.001034  0.001228  0.002395  0.002773  0.175692  0.170225  \n",
       "3    0.001034  0.001823  0.002395  0.002773  0.175692  0.169681  \n",
       "4    0.000641  0.001823  0.002395  0.002773  0.174114  0.169673  \n",
       "..        ...       ...       ...       ...       ...       ...  \n",
       "513  0.001034  0.001823  0.002395  0.002773  0.175692  0.170237  \n",
       "514  0.001034  0.001823  0.002395  0.002773  0.175692  0.170237  \n",
       "515  0.001034  0.001823  0.002395  0.002773  0.175692  0.170237  \n",
       "516  0.001034  0.001823  0.002395  0.002773  0.175692  0.170237  \n",
       "517  0.001034  0.001823  0.002395  0.002773  0.175692  0.170237  \n",
       "\n",
       "[518 rows x 20 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#print(model.p_y_given_x) # n_docs x k_topics\n",
    "\n",
    "claim_topic_probs = pd.DataFrame(model_20.p_y_given_x) # softmax determines label of claim\n",
    "claim_topic_probs\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create full claims dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.003132</td>\n",
       "      <td>0.001405</td>\n",
       "      <td>0.010406</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.004620</td>\n",
       "      <td>0.011083</td>\n",
       "      <td>0.010577</td>\n",
       "      <td>0.00106</td>\n",
       "      <td>0.002529</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.000566</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>0.008207</td>\n",
       "      <td>0.00138</td>\n",
       "      <td>0.001034</td>\n",
       "      <td>0.001823</td>\n",
       "      <td>0.002395</td>\n",
       "      <td>0.002773</td>\n",
       "      <td>0.175692</td>\n",
       "      <td>0.169631</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.003132</td>\n",
       "      <td>0.001405</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.009513</td>\n",
       "      <td>0.004620</td>\n",
       "      <td>0.011083</td>\n",
       "      <td>0.010577</td>\n",
       "      <td>0.00106</td>\n",
       "      <td>0.002529</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.000566</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>0.008207</td>\n",
       "      <td>0.00138</td>\n",
       "      <td>0.000522</td>\n",
       "      <td>0.001823</td>\n",
       "      <td>0.002395</td>\n",
       "      <td>0.002773</td>\n",
       "      <td>0.173437</td>\n",
       "      <td>0.169430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.001405</td>\n",
       "      <td>0.010406</td>\n",
       "      <td>0.009513</td>\n",
       "      <td>0.004620</td>\n",
       "      <td>0.011083</td>\n",
       "      <td>0.010577</td>\n",
       "      <td>0.00106</td>\n",
       "      <td>0.002529</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.000566</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>0.008207</td>\n",
       "      <td>0.00138</td>\n",
       "      <td>0.001034</td>\n",
       "      <td>0.001228</td>\n",
       "      <td>0.002395</td>\n",
       "      <td>0.002773</td>\n",
       "      <td>0.175692</td>\n",
       "      <td>0.170225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.005846</td>\n",
       "      <td>0.001405</td>\n",
       "      <td>0.010406</td>\n",
       "      <td>0.009513</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.011083</td>\n",
       "      <td>0.010577</td>\n",
       "      <td>0.00106</td>\n",
       "      <td>0.002529</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.000566</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>0.008207</td>\n",
       "      <td>0.00138</td>\n",
       "      <td>0.001034</td>\n",
       "      <td>0.001823</td>\n",
       "      <td>0.002395</td>\n",
       "      <td>0.002773</td>\n",
       "      <td>0.175692</td>\n",
       "      <td>0.169681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.001405</td>\n",
       "      <td>0.010406</td>\n",
       "      <td>0.009513</td>\n",
       "      <td>0.006246</td>\n",
       "      <td>0.011083</td>\n",
       "      <td>0.010577</td>\n",
       "      <td>0.00106</td>\n",
       "      <td>0.002529</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.000566</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>0.008207</td>\n",
       "      <td>0.00138</td>\n",
       "      <td>0.000641</td>\n",
       "      <td>0.001823</td>\n",
       "      <td>0.002395</td>\n",
       "      <td>0.002773</td>\n",
       "      <td>0.174114</td>\n",
       "      <td>0.169673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>513</th>\n",
       "      <td>0.003132</td>\n",
       "      <td>0.001405</td>\n",
       "      <td>0.010406</td>\n",
       "      <td>0.009513</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.011083</td>\n",
       "      <td>0.010577</td>\n",
       "      <td>0.00106</td>\n",
       "      <td>0.002529</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.000566</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>0.008207</td>\n",
       "      <td>0.00138</td>\n",
       "      <td>0.001034</td>\n",
       "      <td>0.001823</td>\n",
       "      <td>0.002395</td>\n",
       "      <td>0.002773</td>\n",
       "      <td>0.175692</td>\n",
       "      <td>0.170237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>514</th>\n",
       "      <td>0.003132</td>\n",
       "      <td>0.001405</td>\n",
       "      <td>0.010406</td>\n",
       "      <td>0.009513</td>\n",
       "      <td>0.004620</td>\n",
       "      <td>0.011083</td>\n",
       "      <td>0.010577</td>\n",
       "      <td>0.00106</td>\n",
       "      <td>0.002529</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.000566</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>0.008207</td>\n",
       "      <td>0.00138</td>\n",
       "      <td>0.001034</td>\n",
       "      <td>0.001823</td>\n",
       "      <td>0.002395</td>\n",
       "      <td>0.002773</td>\n",
       "      <td>0.175692</td>\n",
       "      <td>0.170237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>515</th>\n",
       "      <td>0.003132</td>\n",
       "      <td>0.001405</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.009513</td>\n",
       "      <td>0.007108</td>\n",
       "      <td>0.011083</td>\n",
       "      <td>0.010577</td>\n",
       "      <td>0.00106</td>\n",
       "      <td>0.002529</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.000566</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>0.028109</td>\n",
       "      <td>0.00138</td>\n",
       "      <td>0.001034</td>\n",
       "      <td>0.001823</td>\n",
       "      <td>0.002395</td>\n",
       "      <td>0.002773</td>\n",
       "      <td>0.175692</td>\n",
       "      <td>0.170237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>516</th>\n",
       "      <td>0.003132</td>\n",
       "      <td>0.001405</td>\n",
       "      <td>0.020789</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.004620</td>\n",
       "      <td>0.011083</td>\n",
       "      <td>0.010577</td>\n",
       "      <td>0.00106</td>\n",
       "      <td>0.002529</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.000566</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>0.008207</td>\n",
       "      <td>0.00138</td>\n",
       "      <td>0.001034</td>\n",
       "      <td>0.001823</td>\n",
       "      <td>0.002395</td>\n",
       "      <td>0.002773</td>\n",
       "      <td>0.175692</td>\n",
       "      <td>0.170237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>517</th>\n",
       "      <td>0.003132</td>\n",
       "      <td>0.001405</td>\n",
       "      <td>0.010406</td>\n",
       "      <td>0.009513</td>\n",
       "      <td>0.004620</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.010577</td>\n",
       "      <td>0.00106</td>\n",
       "      <td>0.002529</td>\n",
       "      <td>0.000719</td>\n",
       "      <td>0.000566</td>\n",
       "      <td>0.001685</td>\n",
       "      <td>0.008207</td>\n",
       "      <td>0.00138</td>\n",
       "      <td>0.001034</td>\n",
       "      <td>0.001823</td>\n",
       "      <td>0.002395</td>\n",
       "      <td>0.002773</td>\n",
       "      <td>0.175692</td>\n",
       "      <td>0.170237</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>518 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           0         1         2         3         4         5         6   \\\n",
       "0    0.003132  0.001405  0.010406  0.999999  0.004620  0.011083  0.010577   \n",
       "1    0.003132  0.001405  0.999999  0.009513  0.004620  0.011083  0.010577   \n",
       "2    0.999999  0.001405  0.010406  0.009513  0.004620  0.011083  0.010577   \n",
       "3    0.005846  0.001405  0.010406  0.009513  0.999999  0.011083  0.010577   \n",
       "4    0.999999  0.001405  0.010406  0.009513  0.006246  0.011083  0.010577   \n",
       "..        ...       ...       ...       ...       ...       ...       ...   \n",
       "513  0.003132  0.001405  0.010406  0.009513  0.999999  0.011083  0.010577   \n",
       "514  0.003132  0.001405  0.010406  0.009513  0.004620  0.011083  0.010577   \n",
       "515  0.003132  0.001405  0.999999  0.009513  0.007108  0.011083  0.010577   \n",
       "516  0.003132  0.001405  0.020789  0.999999  0.004620  0.011083  0.010577   \n",
       "517  0.003132  0.001405  0.010406  0.009513  0.004620  0.999999  0.010577   \n",
       "\n",
       "          7         8         9         10        11        12       13  \\\n",
       "0    0.00106  0.002529  0.000719  0.000566  0.001685  0.008207  0.00138   \n",
       "1    0.00106  0.002529  0.000719  0.000566  0.001685  0.008207  0.00138   \n",
       "2    0.00106  0.002529  0.000719  0.000566  0.001685  0.008207  0.00138   \n",
       "3    0.00106  0.002529  0.000719  0.000566  0.001685  0.008207  0.00138   \n",
       "4    0.00106  0.002529  0.000719  0.000566  0.001685  0.008207  0.00138   \n",
       "..       ...       ...       ...       ...       ...       ...      ...   \n",
       "513  0.00106  0.002529  0.000719  0.000566  0.001685  0.008207  0.00138   \n",
       "514  0.00106  0.002529  0.000719  0.000566  0.001685  0.008207  0.00138   \n",
       "515  0.00106  0.002529  0.000719  0.000566  0.001685  0.028109  0.00138   \n",
       "516  0.00106  0.002529  0.000719  0.000566  0.001685  0.008207  0.00138   \n",
       "517  0.00106  0.002529  0.000719  0.000566  0.001685  0.008207  0.00138   \n",
       "\n",
       "           14        15        16        17        18        19  \n",
       "0    0.001034  0.001823  0.002395  0.002773  0.175692  0.169631  \n",
       "1    0.000522  0.001823  0.002395  0.002773  0.173437  0.169430  \n",
       "2    0.001034  0.001228  0.002395  0.002773  0.175692  0.170225  \n",
       "3    0.001034  0.001823  0.002395  0.002773  0.175692  0.169681  \n",
       "4    0.000641  0.001823  0.002395  0.002773  0.174114  0.169673  \n",
       "..        ...       ...       ...       ...       ...       ...  \n",
       "513  0.001034  0.001823  0.002395  0.002773  0.175692  0.170237  \n",
       "514  0.001034  0.001823  0.002395  0.002773  0.175692  0.170237  \n",
       "515  0.001034  0.001823  0.002395  0.002773  0.175692  0.170237  \n",
       "516  0.001034  0.001823  0.002395  0.002773  0.175692  0.170237  \n",
       "517  0.001034  0.001823  0.002395  0.002773  0.175692  0.170237  \n",
       "\n",
       "[518 rows x 20 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "claim_topic_probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Raw Claims data, real and fake\n",
    "\n",
    "claim_topic_probs.reset_index(drop=True, inplace=True)\n",
    "claims.reset_index(drop=True, inplace=True)\n",
    "\n",
    "claims['word_count'] = [len(x.split()) for x in claims['title'].tolist()]\n",
    "\n",
    "claims_data = pd.concat([claims, claim_topic_probs], axis = 1)\n",
    "\n",
    "#claims_data.to_csv('claims_data_v2.csv')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sums\n",
    "#sums = claims_data.sum()\n",
    "#sums.to_csv('sums.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}