sentence-embedding-tm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "99c29fac-8168-4f4a-9e37-c74efccc6995",
   "metadata": {},
   "source": [
    "## Text Processing Utilities for City-Data Corpus\n",
    "\n",
    "The functions below were used to normalize the City-Data Corpus texts:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "37f8ac6b-9ef3-450f-baaa-10306caa5faf",
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.cluster import KMeans, SpectralClustering, kmeans_plusplus\n",
    "from sklearn.manifold import SpectralEmbedding\n",
    "from nltk.corpus import stopwords\n",
    "import nltk\n",
    "import re\n",
    "from scipy.cluster.hierarchy import fclusterdata\n",
    "from sklearn.datasets import load_digits\n",
    "from sklearn.feature_selection import SelectKBest, chi2\n",
    "import pickle\n",
    "from gensim.models import TfidfModel\n",
    "from gensim.corpora import Dictionary\n",
    "import pandas as pd\n",
    "import os\n",
    "import re\n",
    "import gensim\n",
    "from collections import Counter\n",
    "import plotly.express as px\n",
    "from plotly.offline import plot\n",
    "from sklearn.cluster import MiniBatchKMeans\n",
    "import pandas as pd\n",
    "import itertools\n",
    "from bs4 import BeautifulSoup as soup\n",
    "from gensim.parsing.preprocessing import remove_stopwords\n",
    "from gensim.parsing.preprocessing import strip_short,strip_non_alphanum, strip_tags,strip_multiple_whitespaces, preprocess_documents, preprocess_string, strip_numeric, remove_stopwords, strip_tags, strip_punctuation, stem_text\n",
    "from gensim.test.utils import datapath\n",
    "from gensim.models.word2vec import Text8Corpus\n",
    "from gensim.models.phrases import Phrases, Phraser\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import networkx as nx\n",
    "\n",
    "def open_pickle(fname):\n",
    "    with open(fname,'rb') as s:\n",
    "        data = pickle.load(s)\n",
    "    return data\n",
    "\n",
    "def save_pickle(f, filename):\n",
    "    with open(filename, 'wb') as h:\n",
    "        pickle.dump(f,h,2)\n",
    "\n",
    "def flatten_list(somelist):\n",
    "    \"\"\"\"\n",
    "    Function to flatten a list of lists.\n",
    "\n",
    "    Args:\n",
    "        somelist: List of lists.\n",
    "\n",
    "    Returns:\n",
    "        Merged list\n",
    "    \"\"\"\n",
    "    if any(isinstance(el, list) for el in somelist) == False:\n",
    "        return somelist\n",
    "    flat_list = list(itertools.chain(*somelist))\n",
    "    return flat_list\n",
    "\n",
    "\n",
    "stops = stopwords.words('english') + ['said','know','maybe','post','advertisements','advertisement','posted','thread','like','could','should','would','thing']\n",
    "wn = WordNetLemmatizer()\n",
    "stemmer = nltk.PorterStemmer()\n",
    "def text_process(text):\n",
    "    \"\"\"\n",
    "    Function to normalize text.\n",
    "\n",
    "    Args:\n",
    "        texts: string\n",
    "\n",
    "    Returns:\n",
    "        string\n",
    "    \"\"\"\n",
    "    \n",
    "    text = text.lower()\n",
    "    tokens = nltk.word_tokenize(text)\n",
    "    tokens = [token for token in tokens if not re.findall(r'\\.com|___|Advertisements|-|_',token)]\n",
    "\n",
    "    return ' '.join([wn.lemmatize(token) for token in tokens if len(token) > 3 and token not in stops and not re.findall(r'[0-9]',token) and not re.findall(r'htt',token)])\n",
    "\n",
    "def make_graph_citydata(dataframe):\n",
    "    \"\"\"\n",
    "    Generates networkx graph of City-Data.com Corpus Forum Data\n",
    "\n",
    "    Args:\n",
    "        dataframe: City-Data.com Corpus Dataframe\n",
    "\n",
    "    Returns:\n",
    "        networkx Graph\n",
    "    \"\"\"\n",
    "    \n",
    "    \n",
    "    \n",
    "    edges = [(str(x),str(y)) for (x,y) in list(zip(dataframe.post_id.tolist(),dataframe.quote_id.tolist()))]\n",
    "    \n",
    "    G = nx.MultiDiGraph()\n",
    "    G.add_edges_from(edges)\n",
    "    try:\n",
    "        G.remove_node('')\n",
    "    except:\n",
    "        pass\n",
    "        \n",
    "    for i in range(len(dataframe)):\n",
    "        G.add_node(str(dataframe.iloc[i]['post_id']),text=dataframe.iloc[i]['post'])\n",
    "                 \n",
    "\n",
    "    for i in range(len(dataframe)):\n",
    "        if dataframe.iloc[i]['quote_id'] != '' and dataframe.iloc[i]['quote_id'] not in G.nodes():\n",
    "            try:\n",
    "                G.add_node(str(dataframe.iloc[i]['quote_id']),text=dataframe.iloc[i]['quote'])\n",
    "            except:\n",
    "                G.add_node(str(dataframe.iloc[i]['quote_id']),text=None)\n",
    "        else:\n",
    "            G.add_node(str(dataframe.iloc[i]['quote_id']),text=None)\n",
    "        \n",
    "    \n",
    "    return G\n",
    "\n",
    "def get_paths_city_data(dataframe):\n",
    "    \"\"\"\n",
    "    Function to extract threaded posts from a Pandas DataFrame.\n",
    "    \n",
    "    Args:\n",
    "        dataframe: City-Data.com Corpus DataFrame\n",
    " \n",
    "    Returns:\n",
    "        networkx graph, list of threads\n",
    "    \"\"\"\n",
    "    G = make_graph_citydata(dataframe)\n",
    "    sink_nodes = [node for node, outdegree in dict(G.out_degree(G.nodes())).items() if outdegree == 0]\n",
    "    source_nodes = [node for node, indegree in dict(G.in_degree(G.nodes())).items() if indegree == 0]\n",
    "    ss_nodes = [(source, sink) for sink in sink_nodes for source in source_nodes]\n",
    "    paths = []\n",
    "    for (source,sink) in ss_nodes:\n",
    "        for path in nx.all_simple_paths(G, source=source, target=sink):\n",
    "            paths.append(path)\n",
    "    return G, paths\n",
    "\n",
    "def make_thread_embeddings(dataframe, model):\n",
    "    \"\"\"\n",
    "    Function to convert City-Data.com Corpus posts and quoted posts into a network graph and embeddings.\n",
    "\n",
    "    Args:\n",
    "        dataframe: City-Data.com Corpus Dataframe\n",
    "        model: sentence-transformer model\n",
    "\n",
    "    Returns:\n",
    "        networkx graph\n",
    "        City-Data.com Corpus threads and singleton posts\n",
    "        City-Data.com Corpus thread and post embeddings\n",
    "    \"\"\"\n",
    "    dataframe.fillna('',inplace=True)\n",
    "    id_text = {}\n",
    "    for i in range(len(dataframe)):\n",
    "        \n",
    "        id_text[dataframe.iloc[i]['quote_id']] = dataframe.iloc[i]['quote']\n",
    "        id_text[dataframe.iloc[i]['post_id']] = dataframe.iloc[i]['post']\n",
    "\n",
    "    G, paths = get_paths_city_data(dataframe)\n",
    "    chains = []\n",
    "    for path in paths:\n",
    "        p = []\n",
    "        for x in path:\n",
    "            try:\n",
    "                p.append(id_text[x])\n",
    "            except:\n",
    "                p.append('')\n",
    "        chains.append(p)\n",
    "    joint_chains = [' '.join(chain) for chain in chains]\n",
    "    embeddings = model.encode(joint_chains)\n",
    "    singletons = [node for node in G.nodes() if node not in flatten_list(paths)]\n",
    "    singleton_embeddings = model.encode([id_text[s] for s in singletons])\n",
    "    singleton_texts = [id_text[s] for s in singletons]\n",
    "    return G, paths, joint_chains, embeddings, singletons, singleton_embeddings,singleton_texts\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "324515b8-ba42-49c7-a345-f362212016e3",
   "metadata": {},
   "source": [
    "## Topical Coherence and Diversity Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "10094e28-91b3-48f1-9ab3-b78a73a21f06",
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.test.utils import common_corpus, common_dictionary\n",
    "from gensim.models.coherencemodel import CoherenceModel\n",
    "from gensim.test.utils import datapath\n",
    "from gensim.models.word2vec import Text8Corpus\n",
    "from gensim.models.phrases import Phrases, Phraser\n",
    "from scipy.spatial import distance\n",
    "from itertools import combinations\n",
    "\n",
    "#Topical Diversity functions are from Terragni (2023)\n",
    "#https://github.com/silviatti/topic-model-diversity\n",
    "def proportion_unique_words(topics, topk=10):\n",
    "    \"\"\"\n",
    "    compute the proportion of unique words\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    topics: a list of lists of words\n",
    "    topk: top k words on which the topic diversity will be computed\n",
    "    \"\"\"\n",
    "    if topk > len(topics[0]):\n",
    "        raise Exception('Words in topics are less than '+str(topk))\n",
    "    else:\n",
    "        unique_words = set()\n",
    "        for topic in topics:\n",
    "            unique_words = unique_words.union(set(topic[:topk]))\n",
    "        puw = len(unique_words) / (topk * len(topics))\n",
    "        return puw\n",
    "\n",
    "\n",
    "def irbo(topics, weight=0.9, topk=10):\n",
    "    \"\"\"\n",
    "    compute the inverted rank-biased overlap\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    topics: a list of lists of words\n",
    "    weight: p (float), default 1.0: Weight of each\n",
    "        agreement at depth d:p**(d-1). When set\n",
    "        to 1.0, there is no weight, the rbo returns\n",
    "        to average overlap.\n",
    "    topk: top k words on which the topic diversity\n",
    "          will be computed\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    irbo : score of the rank biased overlap over the topics\n",
    "    \"\"\"\n",
    "    if topk > len(topics[0]):\n",
    "        raise Exception('Words in topics are less than topk')\n",
    "    else:\n",
    "        collect = []\n",
    "        for list1, list2 in combinations(topics, 2):\n",
    "            word2index = get_word2index(list1, list2)\n",
    "            indexed_list1 = [word2index[word] for word in list1]\n",
    "            indexed_list2 = [word2index[word] for word in list2]\n",
    "            rbo_val = rbo(indexed_list1[:topk], indexed_list2[:topk], p=weight)[2]\n",
    "            collect.append(rbo_val)\n",
    "        return 1 - np.mean(collect)\n",
    "\n",
    "\n",
    "def word_embedding_irbo(topics, word_embedding_model, weight=0.9, topk=10):\n",
    "    '''\n",
    "    compute the word embedding-based inverted rank-biased overlap\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    topics: a list of lists of words\n",
    "    weight: p (float), default 1.0: Weight of each agreement at depth d:\n",
    "    p**(d-1). When set to 1.0, there is no weight, the rbo returns to average overlap.\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    weirbo: word embedding-based inverted rank_biased_overlap over the topics\n",
    "    '''\n",
    "    if topk > len(topics[0]):\n",
    "        raise Exception('Words in topics are less than topk')\n",
    "    else:\n",
    "        collect = []\n",
    "        for list1, list2 in combinations(topics, 2):\n",
    "            word2index = get_word2index(list1, list2)\n",
    "            index2word = {v: k for k, v in word2index.items()}\n",
    "            indexed_list1 = [word2index[word] for word in list1]\n",
    "            indexed_list2 = [word2index[word] for word in list2]\n",
    "            rbo_val = word_embeddings_rbo(indexed_list1[:topk], indexed_list2[:topk], p=weight,\n",
    "                                          index2word=index2word, word2vec=word_embedding_model)[2]\n",
    "            collect.append(rbo_val)\n",
    "        return 1 - np.mean(collect)\n",
    "\n",
    "\n",
    "def pairwise_jaccard_diversity(topics, topk=10):\n",
    "    '''\n",
    "    compute the average pairwise jaccard distance between the topics \n",
    "  \n",
    "    Parameters\n",
    "    ----------\n",
    "    topics: a list of lists of words\n",
    "    topk: top k words on which the topic diversity\n",
    "          will be computed\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    pjd: average pairwise jaccard distance\n",
    "    '''\n",
    "    dist = 0\n",
    "    count = 0\n",
    "    for list1, list2 in combinations(topics, 2):\n",
    "        js = 1 - len(set(list1).intersection(set(list2)))/len(set(list1).union(set(list2)))\n",
    "        dist = dist + js\n",
    "        count = count + 1\n",
    "    return dist/count\n",
    "\n",
    "\n",
    "def pairwise_word_embedding_distance(topics, word_embedding_model, topk=10):\n",
    "    \"\"\"\n",
    "    :param topk: how many most likely words to consider in the evaluation\n",
    "    :return: topic coherence computed on the word embeddings similarities\n",
    "    \"\"\"\n",
    "    if topk > len(topics[0]):\n",
    "        raise Exception('Words in topics are less than topk')\n",
    "    else:\n",
    "        count = 0\n",
    "        sum_dist = 0\n",
    "        for list1, list2 in combinations(topics, 2):\n",
    "            count = count+1\n",
    "            word_counts = 0\n",
    "            dist = 0\n",
    "            for word1 in list1[:topk]:\n",
    "                for word2 in list2[:topk]:\n",
    "                    dist = dist + distance.cosine(word_embedding_model.wv[word1], word_embedding_model.wv[word2])\n",
    "                    word_counts = word_counts + 1\n",
    "\n",
    "            dist = dist/word_counts\n",
    "            sum_dist = sum_dist + dist\n",
    "        return sum_dist/count\n",
    "\n",
    "\n",
    "def centroid_distance(topics, word_embedding_model, topk=10):\n",
    "    \"\"\"\n",
    "    :param topk: how many most likely words to consider in the evaluation\n",
    "    :return: topic coherence computed on the word embeddings similarities\n",
    "    \"\"\"\n",
    "    if topk > len(topics[0]):\n",
    "        raise Exception('Words in topics are less than topk')\n",
    "    else:\n",
    "        count = 0\n",
    "        for list1, list2 in combinations(topics, 2):\n",
    "            count = count + 1\n",
    "            centroid1 = np.zeros(word_embedding_model.vector_size)\n",
    "            centroid2 = np.zeros(word_embedding_model.vector_size)\n",
    "            for word1 in list1[:topk]:\n",
    "                centroid1 = centroid1 + word_embedding_model[word1]\n",
    "            for word2 in list2[:topk]:\n",
    "                centroid2 = centroid2 + word_embedding_model[word2]\n",
    "            centroid1 = centroid1 / len(list1[:topk])\n",
    "            centroid2 = centroid2 / len(list2[:topk])\n",
    "        return distance.cosine(centroid1, centroid2)\n",
    "\n",
    "\n",
    "def get_word2index(list1, list2):\n",
    "    words = set(list1)\n",
    "    words = words.union(set(list2))\n",
    "    word2index = {w: i for i, w in enumerate(words)}\n",
    "    return word2index\n",
    "\"\"\"Rank-biased overlap, a ragged sorted list similarity measure.\n",
    "See http://doi.acm.org/10.1145/1852102.1852106 for details. All functions\n",
    "directly corresponding to concepts from the paper are named so that they can be\n",
    "clearly cross-identified.\n",
    "The definition of overlap has been modified to account for ties. Without this,\n",
    "results for lists with tied items were being inflated. The modification itself\n",
    "is not mentioned in the paper but seems to be reasonable, see function\n",
    "``overlap()``. Places in the code which diverge from the spec in the paper\n",
    "because of this are highlighted with comments.\n",
    "The two main functions for performing an RBO analysis are ``rbo()`` and\n",
    "``rbo_dict()``; see their respective docstrings for how to use them.\n",
    "The following doctest just checks that equivalent specifications of a\n",
    "problem yield the same result using both functions:\n",
    "    >>> lst1 = [{\"c\", \"a\"}, \"b\", \"d\"]\n",
    "    >>> lst2 = [\"a\", {\"c\", \"b\"}, \"d\"]\n",
    "    >>> ans_rbo = _round(rbo(lst1, lst2, p=.9))\n",
    "    >>> dct1 = dict(a=1, b=2, c=1, d=3)\n",
    "    >>> dct2 = dict(a=1, b=2, c=2, d=3)\n",
    "    >>> ans_rbo_dict = _round(rbo_dict(dct1, dct2, p=.9, sort_ascending=True))\n",
    "    >>> ans_rbo == ans_rbo_dict\n",
    "    True\n",
    "\"\"\"\n",
    "\n",
    "import math\n",
    "from bisect import bisect_left\n",
    "from collections import namedtuple\n",
    "\n",
    "\n",
    "RBO = namedtuple(\"RBO\", \"min res ext\")\n",
    "RBO.__doc__ += \": Result of full RBO analysis\"\n",
    "RBO.min.__doc__ = \"Lower bound estimate\"\n",
    "RBO.res.__doc__ = \"Residual corresponding to min; min + res is an upper bound estimate\"\n",
    "RBO.ext.__doc__ = \"Extrapolated point estimate\"\n",
    "\n",
    "\n",
    "def _round(obj):\n",
    "    if isinstance(obj, RBO):\n",
    "        return RBO(_round(obj.min), _round(obj.res), _round(obj.ext))\n",
    "    else:\n",
    "        return round(obj, 3)\n",
    "\n",
    "\n",
    "def set_at_depth(lst, depth):\n",
    "    ans = set()\n",
    "    for v in lst[:depth]:\n",
    "        if isinstance(v, set):\n",
    "            ans.update(v)\n",
    "        else:\n",
    "            ans.add(v)\n",
    "    return ans\n",
    "\n",
    "\n",
    "def raw_overlap(list1, list2, depth):\n",
    "    \"\"\"Overlap as defined in the article.\n",
    "    \"\"\"\n",
    "    set1, set2 = set_at_depth(list1, depth), set_at_depth(list2, depth)\n",
    "    return len(set1.intersection(set2)), len(set1), len(set2)\n",
    "\n",
    "\n",
    "def overlap(list1, list2, depth):\n",
    "    \"\"\"Overlap which accounts for possible ties.\n",
    "    This isn't mentioned in the paper but should be used in the ``rbo*()``\n",
    "    functions below, otherwise overlap at a given depth might be > depth which\n",
    "    inflates the result.\n",
    "    There are no guidelines in the paper as to what's a good way to calculate\n",
    "    this, but a good guess is agreement scaled by the minimum between the\n",
    "    requested depth and the lengths of the considered lists (overlap shouldn't\n",
    "    be larger than the number of ranks in the shorter list, otherwise results\n",
    "    are conspicuously wrong when the lists are of unequal lengths -- rbo_ext is\n",
    "    not between rbo_min and rbo_min + rbo_res.\n",
    "    >>> overlap(\"abcd\", \"abcd\", 3)\n",
    "    3.0\n",
    "    >>> overlap(\"abcd\", \"abcd\", 5)\n",
    "    4.0\n",
    "    >>> overlap([\"a\", {\"b\", \"c\"}, \"d\"], [\"a\", {\"b\", \"c\"}, \"d\"], 2)\n",
    "    2.0\n",
    "    >>> overlap([\"a\", {\"b\", \"c\"}, \"d\"], [\"a\", {\"b\", \"c\"}, \"d\"], 3)\n",
    "    3.0\n",
    "    \"\"\"\n",
    "    ov = agreement(list1, list2, depth) * min(depth, len(list1), len(list2))\n",
    "    return ov\n",
    "    # NOTE: comment the preceding and uncomment the following line if you want\n",
    "    # to stick to the algorithm as defined by the paper\n",
    "    # return raw_overlap(list1, list2, depth)[0]\n",
    "\n",
    "\n",
    "def agreement(list1, list2, depth):\n",
    "    \"\"\"Proportion of shared values between two sorted lists at given depth.\n",
    "    >>> _round(agreement(\"abcde\", \"abdcf\", 1))\n",
    "    1.0\n",
    "    >>> _round(agreement(\"abcde\", \"abdcf\", 3))\n",
    "    0.667\n",
    "    >>> _round(agreement(\"abcde\", \"abdcf\", 4))\n",
    "    1.0\n",
    "    >>> _round(agreement(\"abcde\", \"abdcf\", 5))\n",
    "    0.8\n",
    "    >>> _round(agreement([{1, 2}, 3], [1, {2, 3}], 1))\n",
    "    0.667\n",
    "    >>> _round(agreement([{1, 2}, 3], [1, {2, 3}], 2))\n",
    "    1.0\n",
    "    \"\"\"\n",
    "    len_intersection, len_set1, len_set2 = raw_overlap(list1, list2, depth)\n",
    "    return 2 * len_intersection / (len_set1 + len_set2)\n",
    "\n",
    "\n",
    "def cumulative_agreement(list1, list2, depth):\n",
    "    return (agreement(list1, list2, d) for d in range(1, depth + 1))\n",
    "\n",
    "\n",
    "def average_overlap(list1, list2, depth=None):\n",
    "    \"\"\"Calculate average overlap between ``list1`` and ``list2``.\n",
    "    >>> _round(average_overlap(\"abcdefg\", \"zcavwxy\", 1))\n",
    "    0.0\n",
    "    >>> _round(average_overlap(\"abcdefg\", \"zcavwxy\", 2))\n",
    "    0.0\n",
    "    >>> _round(average_overlap(\"abcdefg\", \"zcavwxy\", 3))\n",
    "    0.222\n",
    "    >>> _round(average_overlap(\"abcdefg\", \"zcavwxy\", 4))\n",
    "    0.292\n",
    "    >>> _round(average_overlap(\"abcdefg\", \"zcavwxy\", 5))\n",
    "    0.313\n",
    "    >>> _round(average_overlap(\"abcdefg\", \"zcavwxy\", 6))\n",
    "    0.317\n",
    "    >>> _round(average_overlap(\"abcdefg\", \"zcavwxy\", 7))\n",
    "    0.312\n",
    "    \"\"\"\n",
    "    depth = min(len(list1), len(list2)) if depth is None else depth\n",
    "    return sum(cumulative_agreement(list1, list2, depth)) / depth\n",
    "\n",
    "\n",
    "def rbo_at_k(list1, list2, p, depth=None):\n",
    "    # ``p**d`` here instead of ``p**(d - 1)`` because enumerate starts at\n",
    "    # 0\n",
    "    depth = min(len(list1), len(list2)) if depth is None else depth\n",
    "    d_a = enumerate(cumulative_agreement(list1, list2, depth))\n",
    "    return (1 - p) * sum(p ** d * a for (d, a) in d_a)\n",
    "\n",
    "\n",
    "def rbo_min(list1, list2, p, depth=None):\n",
    "    \"\"\"Tight lower bound on RBO.\n",
    "    See equation (11) in paper.\n",
    "    >>> _round(rbo_min(\"abcdefg\", \"abcdefg\", .9))\n",
    "    0.767\n",
    "    >>> _round(rbo_min(\"abcdefgh\", \"abcdefg\", .9))\n",
    "    0.767\n",
    "    \"\"\"\n",
    "    depth = min(len(list1), len(list2)) if depth is None else depth\n",
    "    x_k = overlap(list1, list2, depth)\n",
    "    log_term = x_k * math.log(1 - p)\n",
    "    sum_term = sum(\n",
    "        p ** d / d * (overlap(list1, list2, d) - x_k) for d in range(1, depth + 1)\n",
    "    )\n",
    "    return (1 - p) / p * (sum_term - log_term)\n",
    "\n",
    "\n",
    "def rbo_res(list1, list2, p):\n",
    "    \"\"\"Upper bound on residual overlap beyond evaluated depth.\n",
    "    See equation (30) in paper.\n",
    "    NOTE: The doctests weren't verified against manual computations but seem\n",
    "    plausible. In particular, for identical lists, ``rbo_min()`` and\n",
    "    ``rbo_res()`` should add up to 1, which is the case.\n",
    "    >>> _round(rbo_res(\"abcdefg\", \"abcdefg\", .9))\n",
    "    0.233\n",
    "    >>> _round(rbo_res(\"abcdefg\", \"abcdefghijklmnopqrstuvwxyz\", .9))\n",
    "    0.239\n",
    "    \"\"\"\n",
    "    S, L = sorted((list1, list2), key=len)\n",
    "    s, l = len(S), len(L)\n",
    "    x_l = overlap(list1, list2, l)\n",
    "    # since overlap(...) can be fractional in the general case of ties and f\n",
    "    # must be an integer --> math.ceil()\n",
    "    f = int(math.ceil(l + s - x_l))\n",
    "    # upper bound of range() is non-inclusive, therefore + 1 is needed\n",
    "    term1 = s * sum(p ** d / d for d in range(s + 1, f + 1))\n",
    "    term2 = l * sum(p ** d / d for d in range(l + 1, f + 1))\n",
    "    term3 = x_l * (math.log(1 / (1 - p)) - sum(p ** d / d for d in range(1, f + 1)))\n",
    "    return p ** s + p ** l - p ** f - (1 - p) / p * (term1 + term2 + term3)\n",
    "\n",
    "\n",
    "def rbo_ext(list1, list2, p):\n",
    "    \"\"\"RBO point estimate based on extrapolating observed overlap.\n",
    "    See equation (32) in paper.\n",
    "    NOTE: The doctests weren't verified against manual computations but seem\n",
    "    plausible.\n",
    "    >>> _round(rbo_ext(\"abcdefg\", \"abcdefg\", .9))\n",
    "    1.0\n",
    "    >>> _round(rbo_ext(\"abcdefg\", \"bacdefg\", .9))\n",
    "    0.9\n",
    "    \"\"\"\n",
    "    S, L = sorted((list1, list2), key=len)\n",
    "    s, l = len(S), len(L)\n",
    "    x_l = overlap(list1, list2, l)\n",
    "    x_s = overlap(list1, list2, s)\n",
    "    # the paper says overlap(..., d) / d, but it should be replaced by\n",
    "    # agreement(..., d) defined as per equation (28) so that ties are handled\n",
    "    # properly (otherwise values > 1 will be returned)\n",
    "    # sum1 = sum(p**d * overlap(list1, list2, d)[0] / d for d in range(1, l + 1))\n",
    "    sum1 = sum(p ** d * agreement(list1, list2, d) for d in range(1, l + 1))\n",
    "    sum2 = sum(p ** d * x_s * (d - s) / s / d for d in range(s + 1, l + 1))\n",
    "    term1 = (1 - p) / p * (sum1 + sum2)\n",
    "    term2 = p ** l * ((x_l - x_s) / l + x_s / s)\n",
    "    return term1 + term2\n",
    "\n",
    "\n",
    "def rbo(list1, list2, p):\n",
    "    \"\"\"Complete RBO analysis (lower bound, residual, point estimate).\n",
    "    ``list`` arguments should be already correctly sorted iterables and each\n",
    "    item should either be an atomic value or a set of values tied for that\n",
    "    rank. ``p`` is the probability of looking for overlap at rank k + 1 after\n",
    "    having examined rank k.\n",
    "    >>> lst1 = [{\"c\", \"a\"}, \"b\", \"d\"]\n",
    "    >>> lst2 = [\"a\", {\"c\", \"b\"}, \"d\"]\n",
    "    >>> _round(rbo(lst1, lst2, p=.9))\n",
    "    RBO(min=0.489, res=0.477, ext=0.967)\n",
    "    \"\"\"\n",
    "    if not 0 <= p <= 1:\n",
    "        raise ValueError(\"The ``p`` parameter must be between 0 and 1.\")\n",
    "    args = (list1, list2, p)\n",
    "    return RBO(rbo_min(*args), rbo_res(*args), rbo_ext(*args))\n",
    "\n",
    "\n",
    "def sort_dict(dct, *, ascending=False):\n",
    "    \"\"\"Sort keys in ``dct`` according to their corresponding values.\n",
    "    Sorts in descending order by default, because the values are\n",
    "    typically scores, i.e. the higher the better. Specify\n",
    "    ``ascending=True`` if the values are ranks, or some sort of score\n",
    "    where lower values are better.\n",
    "    Ties are handled by creating sets of tied keys at the given position\n",
    "    in the sorted list.\n",
    "    >>> dct = dict(a=1, b=2, c=1, d=3)\n",
    "    >>> list(sort_dict(dct)) == ['d', 'b', {'a', 'c'}]\n",
    "    True\n",
    "    >>> list(sort_dict(dct, ascending=True)) == [{'a', 'c'}, 'b', 'd']\n",
    "    True\n",
    "    \"\"\"\n",
    "    scores = []\n",
    "    items = []\n",
    "    # items should be unique, scores don't have to\n",
    "    for item, score in dct.items():\n",
    "        if not ascending:\n",
    "            score *= -1\n",
    "        i = bisect_left(scores, score)\n",
    "        if i == len(scores):\n",
    "            scores.append(score)\n",
    "            items.append(item)\n",
    "        elif scores[i] == score:\n",
    "            existing_item = items[i]\n",
    "            if isinstance(existing_item, set):\n",
    "                existing_item.add(item)\n",
    "            else:\n",
    "                items[i] = {existing_item, item}\n",
    "        else:\n",
    "            scores.insert(i, score)\n",
    "            items.insert(i, item)\n",
    "    return items\n",
    "\n",
    "\n",
    "def rbo_dict(dict1, dict2, p, *, sort_ascending=False):\n",
    "    \"\"\"Wrapper around ``rbo()`` for dict input.\n",
    "    Each dict maps items to be sorted to the score according to which\n",
    "    they should be sorted. The RBO analysis is then performed on the\n",
    "    resulting sorted lists.\n",
    "    The sort is descending by default, because scores are typically the\n",
    "    higher the better, but this can be overridden by specifying\n",
    "    ``sort_ascending=True``.\n",
    "    >>> dct1 = dict(a=1, b=2, c=1, d=3)\n",
    "    >>> dct2 = dict(a=1, b=2, c=2, d=3)\n",
    "    >>> _round(rbo_dict(dct1, dct2, p=.9, sort_ascending=True))\n",
    "    RBO(min=0.489, res=0.477, ext=0.967)\n",
    "    \"\"\"\n",
    "    list1, list2 = (\n",
    "        sort_dict(dict1, ascending=sort_ascending),\n",
    "        sort_dict(dict2, ascending=sort_ascending),\n",
    "    )\n",
    "    return rbo(list1, list2, p)\n",
    "\n",
    "import math\n",
    "from bisect import bisect_left\n",
    "from collections import namedtuple\n",
    "from collections import OrderedDict\n",
    "\n",
    "RBO = namedtuple(\"RBO\", \"min res ext\")\n",
    "RBO.__doc__ += \": Result of full RBO analysis\"\n",
    "RBO.min.__doc__ = \"Lower bound estimate\"\n",
    "RBO.res.__doc__ = \"Residual corresponding to min; min + res is an upper bound estimate\"\n",
    "RBO.ext.__doc__ = \"Extrapolated point estimate\"\n",
    "\n",
    "def _round(obj):\n",
    "    if isinstance(obj, RBO):\n",
    "        return RBO(_round(obj.min), _round(obj.res), _round(obj.ext))\n",
    "    else:\n",
    "        return round(obj, 3)\n",
    "\n",
    "\n",
    "def set_at_depth(lst, depth):\n",
    "    ans = set()\n",
    "    for v in lst[:depth]:\n",
    "        if isinstance(v, set):\n",
    "            ans.update(v)\n",
    "        else:\n",
    "            ans.add(v)\n",
    "    return ans\n",
    "\n",
    "\n",
    "def embeddings_overlap(list1, list2, depth, index2word, word2vec):\n",
    "    #set1, set2 = set_at_depth(list1, depth), set_at_depth(list2, depth)\n",
    "    #return len(set1.intersection(set2)), len(set1), len(set2)\n",
    "\n",
    "    set1, set2 = set_at_depth(list1, depth), set_at_depth(list2, depth)\n",
    "    word_list1 = [index2word[index] for index in list1]\n",
    "    word_list2 = [index2word[index] for index in list2]\n",
    "\n",
    "    similarities = {}\n",
    "    for w1 in word_list1[:depth]:\n",
    "        for w2 in word_list2[:depth]:\n",
    "            similarities[(w1,w2)] = word2vec.similarity(w1, w2)\n",
    "\n",
    "    similarities = OrderedDict(sorted(similarities.items(), key=lambda x: -x[1]))\n",
    "\n",
    "    e_ov = 0\n",
    "    key_list = list(similarities.keys())\n",
    "    for k in key_list:\n",
    "        if k in similarities.keys():\n",
    "            #print(k, similarities[k])\n",
    "            e_ov = e_ov + similarities[k]\n",
    "            similarities = {save_k: v for save_k, v in similarities.items()\n",
    "                            if save_k[0] != k[0] and save_k[1] != k[1]}\n",
    "    #e_ov = 1\n",
    "    #print(\"****\")\n",
    "    return e_ov, len(set1), len(set2)\n",
    "\n",
    "\n",
    "def overlap(list1, list2, depth, index2word, word2vec):\n",
    "    #return agreement(list1, list2, depth) * min(depth, len(list1), len(list2))\n",
    "    # NOTE: comment the preceding and uncomment the following line if you want\n",
    "    # to stick to the algorithm as defined by the paper\n",
    "    ov = embeddings_overlap(list1, list2, depth, index2word, word2vec)[0]\n",
    "    return ov\n",
    "\n",
    "\n",
    "def agreement(list1, list2, depth, index2word, word2vec):\n",
    "    \"\"\"Proportion of shared values between two sorted lists at given depth.\"\"\"\n",
    "    len_intersection, len_set1, len_set2 = embeddings_overlap(list1, list2, depth, index2word, word2vec)\n",
    "    return 2 * len_intersection / (len_set1 + len_set2)\n",
    "\n",
    "\n",
    "def cumulative_agreement(list1, list2, depth, index2word, word2vec):\n",
    "    return (agreement(list1, list2, d, index2word, word2vec) for d in range(1, depth + 1))\n",
    "\n",
    "\n",
    "def average_overlap(list1, list2, index2word, word2vec, depth=None):\n",
    "    \"\"\"Calculate average overlap between ``list1`` and ``list2``.\n",
    "    \"\"\"\n",
    "    depth = min(len(list1), len(list2)) if depth is None else depth\n",
    "    return sum(cumulative_agreement(list1, list2, depth, index2word=index2word, word2vec=word2vec)) / depth\n",
    "\n",
    "\n",
    "def rbo_at_k(list1, list2, p, index2word, word2vec, depth=None):\n",
    "    # ``p**d`` here instead of ``p**(d - 1)`` because enumerate starts at\n",
    "    # 0\n",
    "    depth = min(len(list1), len(list2)) if depth is None else depth\n",
    "    d_a = enumerate(cumulative_agreement(list1, list2, depth, index2word=index2word, word2vec=word2vec))\n",
    "    return (1 - p) * sum(p ** d * a for (d, a) in d_a)\n",
    "\n",
    "\n",
    "def rbo_min(list1, list2, p, index2word, word2vec, depth=None):\n",
    "    \"\"\"Tight lower bound on RBO.\n",
    "    See equation (11) in paper.\n",
    "    \"\"\"\n",
    "    depth = min(len(list1), len(list2)) if depth is None else depth\n",
    "    x_k = overlap(list1, list2, depth, index2word, word2vec)\n",
    "    log_term = x_k * math.log(1 - p)\n",
    "    sum_term = sum(\n",
    "        p ** d / d * (overlap(list1, list2, d, index2word, word2vec=word2vec) - x_k) for d in range(1, depth + 1)\n",
    "    )\n",
    "    return (1 - p) / p * (sum_term - log_term)\n",
    "\n",
    "\n",
    "def rbo_res(list1, list2, p, index2word, word2vec):\n",
    "    \"\"\"Upper bound on residual overlap beyond evaluated depth.\n",
    "    See equation (30) in paper.\n",
    "    NOTE: The doctests weren't verified against manual computations but seem\n",
    "    plausible. In particular, for identical lists, ``rbo_min()`` and\n",
    "    ``rbo_res()`` should add up to 1, which is the case.\n",
    "    \"\"\"\n",
    "    S, L = sorted((list1, list2), key=len)\n",
    "    s, l = len(S), len(L)\n",
    "    x_l = overlap(list1, list2, l, index2word, word2vec)\n",
    "    # since overlap(...) can be fractional in the general case of ties and f\n",
    "    # must be an integer --> math.ceil()\n",
    "    f = int(math.ceil(l + s - x_l))\n",
    "    # upper bound of range() is non-inclusive, therefore + 1 is needed\n",
    "    term1 = s * sum(p ** d / d for d in range(s + 1, f + 1))\n",
    "    term2 = l * sum(p ** d / d for d in range(l + 1, f + 1))\n",
    "    term3 = x_l * (math.log(1 / (1 - p)) - sum(p ** d / d for d in range(1, f + 1)))\n",
    "    return p ** s + p ** l - p ** f - (1 - p) / p * (term1 + term2 + term3)\n",
    "\n",
    "\n",
    "def rbo_ext(list1, list2, p, index2word, word2vec):\n",
    "    \"\"\"RBO point estimate based on extrapolating observed overlap.\n",
    "    See equation (32) in paper.\n",
    "    NOTE: The doctests weren't verified against manual computations but seem\n",
    "    plausible.\n",
    "    >>> _round(rbo_ext(\"abcdefg\", \"abcdefg\", .9))\n",
    "    1.0\n",
    "    >>> _round(rbo_ext(\"abcdefg\", \"bacdefg\", .9))\n",
    "    0.9\n",
    "    \"\"\"\n",
    "    S, L = sorted((list1, list2), key=len)\n",
    "    s, l = len(S), len(L)\n",
    "    x_l = overlap(list1, list2, l, index2word, word2vec)\n",
    "    x_s = overlap(list1, list2, s, index2word, word2vec)\n",
    "    # the paper says overlap(..., d) / d, but it should be replaced by\n",
    "    # agreement(..., d) defined as per equation (28) so that ties are handled\n",
    "    # properly (otherwise values > 1 will be returned)\n",
    "    # sum1 = sum(p**d * overlap(list1, list2, d)[0] / d for d in range(1, l + 1))\n",
    "    sum1 = sum(p ** d * agreement(list1, list2, d, index2word=index2word, word2vec=word2vec)\n",
    "               for d in range(1, l + 1))\n",
    "    sum2 = sum(p ** d * x_s * (d - s) / s / d for d in range(s + 1, l + 1))\n",
    "    term1 = (1 - p) / p * (sum1 + sum2)\n",
    "    term2 = p ** l * ((x_l - x_s) / l + x_s / s)\n",
    "    return term1 + term2\n",
    "\n",
    "\n",
    "def word_embeddings_rbo(list1, list2, p, index2word, word2vec):\n",
    "    \"\"\"Complete RBO analysis (lower bound, residual, point estimate).\n",
    "    ``list`` arguments should be already correctly sorted iterables and each\n",
    "    item should either be an atomic value or a set of values tied for that\n",
    "    rank. ``p`` is the probability of looking for overlap at rank k + 1 after\n",
    "    having examined rank k.\n",
    "    >>> lst1 = [{\"c\", \"a\"}, \"b\", \"d\"]\n",
    "    >>> lst2 = [\"a\", {\"c\", \"b\"}, \"d\"]\n",
    "    >>> _round(rbo(lst1, lst2, p=.9))\n",
    "    RBO(min=0.489, res=0.477, ext=0.967)\n",
    "    \"\"\"\n",
    "    if not 0 <= p <= 1:\n",
    "        raise ValueError(\"The ``p`` parameter must be between 0 and 1.\")\n",
    "    args = (list1, list2, p, index2word, word2vec)\n",
    "\n",
    "    return RBO(rbo_min(*args), rbo_res(*args), rbo_ext(*args))\n",
    "\n",
    "\n",
    "def sort_dict(dct, *, ascending=False):\n",
    "    \"\"\"Sort keys in ``dct`` according to their corresponding values.\n",
    "    Sorts in descending order by default, because the values are\n",
    "    typically scores, i.e. the higher the better. Specify\n",
    "    ``ascending=True`` if the values are ranks, or some sort of score\n",
    "    where lower values are better.\n",
    "    Ties are handled by creating sets of tied keys at the given position\n",
    "    in the sorted list.\n",
    "    >>> dct = dict(a=1, b=2, c=1, d=3)\n",
    "    >>> list(sort_dict(dct)) == ['d', 'b', {'a', 'c'}]\n",
    "    True\n",
    "    >>> list(sort_dict(dct, ascending=True)) == [{'a', 'c'}, 'b', 'd']\n",
    "    True\n",
    "    \"\"\"\n",
    "    scores = []\n",
    "    items = []\n",
    "    # items should be unique, scores don't have to\n",
    "    for item, score in dct.items():\n",
    "        if not ascending:\n",
    "            score *= -1\n",
    "        i = bisect_left(scores, score)\n",
    "        if i == len(scores):\n",
    "            scores.append(score)\n",
    "            items.append(item)\n",
    "        elif scores[i] == score:\n",
    "            existing_item = items[i]\n",
    "            if isinstance(existing_item, set):\n",
    "                existing_item.add(item)\n",
    "            else:\n",
    "                items[i] = {existing_item, item}\n",
    "        else:\n",
    "            scores.insert(i, score)\n",
    "            items.insert(i, item)\n",
    "    return items\n",
    "\n",
    "\n",
    "def rbo_dict(dict1, dict2, p, index2word, word2vec, *, sort_ascending=False):\n",
    "    \"\"\"Wrapper around ``rbo()`` for dict input.\n",
    "    Each dict maps items to be sorted to the score according to which\n",
    "    they should be sorted. The RBO analysis is then performed on the\n",
    "    resulting sorted lists.\n",
    "    The sort is descending by default, because scores are typically the\n",
    "    higher the better, but this can be overridden by specifying\n",
    "    ``sort_ascending=True``.\n",
    "    \"\"\"\n",
    "    list1, list2 = (\n",
    "        sort_dict(dict1, ascending=sort_ascending),\n",
    "        sort_dict(dict2, ascending=sort_ascending),\n",
    "    )\n",
    "    return word_embeddings_rbo(list1, list2, p, index2word, word2vec)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "f451e02b-81c6-4b61-9031-c2270ce59521",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Wrapper code for Terragni's (2023) Topical Diversity Measures\n",
    "\n",
    "\n",
    "def get_topical_coherence(texts,topics, metric='u_mass'):\n",
    "    if type(texts[0]) == str:\n",
    "        #train dictionary on tokenized documents (corpus)\n",
    "        doc_tokens = [text_tokenize(text) for text in texts]\n",
    "        phrases = Phrases(doc_tokens, min_count=1, threshold=1)\n",
    "        bigrams = phrases[doc_tokens]\n",
    "        dct = Dictionary(bigrams) \n",
    "        \n",
    "        \n",
    "        #topics = list of topical words\n",
    "        #texts = tokenized documents\n",
    "        #corpus = doc2bow\n",
    "        #dictionary = gensim dictionary\n",
    "        cm = CoherenceModel(topics=topics,texts=texts, corpus=[dct.doc2bow(c) for c in doc_tokens], dictionary=dct, coherence=metric)\n",
    "        return cm.get_coherence()\n",
    "    elif type(texts[0]) == list:\n",
    "        doc_tokens = texts\n",
    "        phrases = Phrases(doc_tokens, min_count=1, threshold=1)\n",
    "        bigrams = phrases[doc_tokens]\n",
    "        dct = Dictionary(bigrams) \n",
    "        \n",
    "        \n",
    "        #topics = list of topical words\n",
    "        #texts = tokenized documents\n",
    "        #corpus = doc2bow\n",
    "        #dictionary = gensim dictionary\n",
    "        cm = CoherenceModel(topics=topics,texts=texts, corpus=[dct.doc2bow(c) for c in doc_tokens], dictionary=dct, coherence=metric)\n",
    "        return cm.get_coherence()\n",
    "        \n",
    "\n",
    "def get_topical_terms(dataframe, term_count=25):\n",
    "    topics = []\n",
    "    for i in range(len(dataframe.columns)-1):\n",
    "        \n",
    "        terms = dataframe.sort_values(by='coef_'+str(i),ascending=False)[:term_count]['term'].tolist()\n",
    "        topics.append(terms)\n",
    "    filtered = []\n",
    "    for topic in topics:\n",
    "        t = []\n",
    "        for term in topic:\n",
    "            if len(term.split()) > 1:\n",
    "                t.append('_'.join(term.split()))\n",
    "            else:\n",
    "                t.append(term)\n",
    "        filtered.append(t)\n",
    "    return filtered\n",
    "\n",
    "def get_diversity_scores(topics, model, topn=10, topic_type=''):\n",
    "    df = pd.DataFrame({\"puw:\":proportion_unique_words(topics, topk=topn),\n",
    "        \"jd:\": pairwise_jaccard_diversity(topics, topk=topn),\n",
    "        \"we-pd:\": pairwise_word_embedding_distance(topics, model, topk=topn),\n",
    "        \"we-cd:\": centroid_distance(topics, model.wv, topk=topn),\n",
    "        \"we-irbo p=0.5:\":word_embedding_irbo(topics,model.wv, weight=0.5, topk=topn),\n",
    "        \"we-irbo p=0.9:\":word_embedding_irbo(topics,model.wv, weight=0.9, topk=topn)},index=[topic_type])\n",
    "    return df\n",
    "\n",
    "def get_coherence_diversity_scores(texts, topics, model, topn=10, topic_type='',metric='u_mass'):\n",
    "    df = pd.DataFrame({'u_mass_coherence':get_topical_coherence(texts,topics,metric=metric),\n",
    "        \"puw:\":proportion_unique_words(topics, topk=topn),\n",
    "        \"jd:\": pairwise_jaccard_diversity(topics, topk=topn),\n",
    "        \"we-pd:\": pairwise_word_embedding_distance(topics, model, topk=topn),\n",
    "        \"we-cd:\": centroid_distance(topics, model.wv, topk=topn),\n",
    "        \"we-irbo p=0.5:\":word_embedding_irbo(topics,model.wv, weight=0.5, topk=topn),\n",
    "        \"we-irbo p=0.9:\":word_embedding_irbo(topics,model.wv, weight=0.9, topk=topn)},index=[topic_type])\n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e89f2a30-5e09-4d49-a697-74b302607dcb",
   "metadata": {},
   "source": [
    "## Gensim LDA Modeling Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "19b8b70d-2c04-4fbb-819d-e8daeda2011c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim import models\n",
    "def gensim_lda(texts, topic_num=5, topic_word_priors=None,numwords=25, eta_=None, tfidf=False):\n",
    "    \"\"\"\n",
    "    Gensim lda wrapper with guided topic modeling.\n",
    "\n",
    "    Args:\n",
    "        texts: list of strings\n",
    "        topic_num: (int) number of topics\n",
    "        topic_word_priors: list of words (string) to guide modeling\n",
    "        numwords: (int) number of topical terms\n",
    "        eta_: None or list of ints\n",
    "        tfidf: (bool) if True, then use gensim tfidf term weighting (default is False)\n",
    "        \n",
    "    \"\"\"\n",
    "    \n",
    "    if tfidf != False:\n",
    "        #process texts\n",
    "        #build gensim dictionary\n",
    "        processed_texts = [text_tokenize(text) for text in texts]\n",
    "        dictionary = gensim.corpora.Dictionary(processed_texts)\n",
    "    \n",
    "        #build bag-of-words representation\n",
    "        bow = [dictionary.doc2bow(text.split()) for text in texts]\n",
    "        tfidf = models.TfidfModel(bow)\n",
    "        corpus_tfidf = tfidf[bow]\n",
    "        #guided lda with eta\n",
    "        if topic_word_priors and eta_ != None:\n",
    "            etas = []\n",
    "        \n",
    "            for r in range(len(topic_word_priors)):\n",
    "                eta = []\n",
    "                for i in range(len(dictionary)):\n",
    "                    \n",
    "                    if dictionary[i] in topic_word_priors[r]:\n",
    "                        eta.append(np.array(eta_))\n",
    "                    else:\n",
    "                        eta.append(np.array(1/topic_num))\n",
    "                etas.append(eta)\n",
    "    \n",
    "            model = gensim.models.ldamodel.LdaModel(\n",
    "                corpus=corpus_tfidf, id2word=dictionary, num_topics=topic_num,\n",
    "                random_state=42, chunksize=100, eta=np.array(etas),\n",
    "                eval_every=-1, update_every=1,\n",
    "                passes=150, alpha='auto', per_word_topics=True)\n",
    "    \n",
    "            #transform corpus into topics\n",
    "            transformed_corpus = model[corpus_tfidf]\n",
    "    \n",
    "            #extract topical terms\n",
    "            topical_terms = [[token for token in m[1].split('\"') if not re.findall(r'\\d',token)] for m in  model.show_topics(num_words=numwords)]\n",
    "            return model, topical_terms, transformed_corpus, processed_texts\n",
    "            \n",
    "        else:\n",
    "            #standard lda\n",
    "            model = gensim.models.ldamodel.LdaModel(\n",
    "                corpus=corpus_tfidf, id2word=dictionary, num_topics=topic_num,\n",
    "                random_state=42, chunksize=100, eta=None,\n",
    "                eval_every=-1, update_every=1,\n",
    "                passes=150, alpha='auto', per_word_topics=True)\n",
    "            \n",
    "            #transform corpus into topics\n",
    "            transformed_corpus = model[corpus_tfidf]\n",
    "            topical_terms = [[token for token in m[1].split('\"') if not re.findall(r'\\d',token)] for m in  model.show_topics(num_words=numwords)]\n",
    "            return model, topical_terms, transformed_corpus, processed_texts\n",
    "    else:\n",
    "        #process texts\n",
    "        #build gensim dictionary\n",
    "        processed_texts = [text_tokenize(text) for text in texts]\n",
    "        dictionary = gensim.corpora.Dictionary(processed_texts)\n",
    "        \n",
    "        #build bag-of-words representation\n",
    "        bow = [dictionary.doc2bow(text.split()) for text in texts]\n",
    "        #tfidf = models.TfidfModel(bow)\n",
    "        #corpus_tfidf = tfidf[bow]\n",
    "    \n",
    "    \n",
    "        #guided lda with eta\n",
    "        if topic_word_priors and eta_ != None:\n",
    "            etas = []\n",
    "        \n",
    "            for r in range(len(topic_word_priors)):\n",
    "                eta = []\n",
    "                for i in range(len(dictionary)):\n",
    "                    \n",
    "                    if dictionary[i] in topic_word_priors[r]:\n",
    "                        eta.append(np.array(eta_))\n",
    "                    else:\n",
    "                        eta.append(np.array(1/topic_num))\n",
    "                etas.append(eta)\n",
    "    \n",
    "            model = gensim.models.ldamodel.LdaModel(\n",
    "                corpus=bow, id2word=dictionary, num_topics=topic_num,\n",
    "                random_state=42, chunksize=100, eta=np.array(etas),\n",
    "                eval_every=-1, update_every=1,\n",
    "                passes=150, alpha='auto', per_word_topics=True)\n",
    "    \n",
    "            #transform corpus into topics\n",
    "            transformed_corpus = model[bow]\n",
    "    \n",
    "            #extract topical terms\n",
    "            topical_terms = [[token for token in m[1].split('\"') if not re.findall(r'\\d',token)] for m in  model.show_topics(num_words=numwords)]\n",
    "            return model, topical_terms, transformed_corpus, processed_texts\n",
    "        \n",
    "        else:\n",
    "        #standard lda\n",
    "            model = gensim.models.ldamodel.LdaModel(\n",
    "                corpus=bow, id2word=dictionary, num_topics=topic_num,\n",
    "                random_state=42, chunksize=100, eta=None,\n",
    "                eval_every=-1, update_every=1,\n",
    "                passes=150, alpha='auto', per_word_topics=True)\n",
    "            \n",
    "            #transform corpus into topics\n",
    "            transformed_corpus = model[bow]\n",
    "            \n",
    "            #extract topical terms\n",
    "            topical_terms = [[token for token in m[1].split('\"') if not re.findall(r'\\d',token)] for m in  model.show_topics(num_words=25)]\n",
    "            return model, topical_terms, transformed_corpus, processed_texts\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30bdc0b3-1719-4031-88d9-28dc8d9349e2",
   "metadata": {},
   "source": [
    "## City-Data.com Corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "9c8688cc-b97b-47e2-87e6-a3f4770dfd1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#load City-Data.com Corpus from Zenodo\n",
    "citydata = pd.read_csv(\"https://zenodo.org/records/10086354/files/citydata.csv?download=1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "4856a32f-c739-456d-96f2-ef051219e446",
   "metadata": {},
   "outputs": [],
   "source": [
    "#extract all City-Data.com Corpus posts\n",
    "\n",
    "citydata.fillna('',inplace=True) #remove NaN\n",
    "posts = citydata.post.tolist() + [quote for quote in citydata.quote.tolist() if quote not in citydata.post.tolist()]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "24bf5dee-1741-4669-80b1-63cc4fe4b856",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\" So just about everything is shutting down slowly but surely.  My job has shut down for the rest of the month so I'll be home.  lol still haven't gotten to the grocery store.  I'm in PT after a knee replacement and PT is considered an essential operation so still going to that.Stay well guys\",\n",
       " \"I am so lucky to be able to work remotely through something like this.  I feel bad for all the service workers and business people who could lose their jobs.  My family has been taking it easy.  We cook/bake most meals from home and have a huge pantry so we're comfortable and operating mostly normal.  Go out for walks, relax, kids nap, watch movies.  Talking to neighbors and keeping calm/reasonable.  Good luck on your knee rehab.  Good luck to everyone!\",\n",
       " 'I agree with really feeling for those in the service and medical industries, as well as the many people who canâ€™t stay home. I hope companies with employees that canâ€™t work remotely have a plan for anyone 60+ to receive paid time off until this is settled.Right now Iâ€™m fine with social distancing, Iâ€™m just worried about how sustainable this is long-term. I went to the Wegmans in KoP this morning, and although I dodged the crowds, the store was well cleared of many essentials.',\n",
       " \"I'm working remotely as well. Went for booze yesterday.I'm a total hypochondraic though so I'm convinced I have it. No symptoms of course.\",\n",
       " \"Working remotely for the foreseeable future. We have food (human and canine) for now and since it's really only Day 1, no one is going stir-crazy yet. But that's what the elliptical and walks outside (alone, of course, lol) are for. Having a dog is helpful, because it forces you to go outside.\"]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "posts[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cefec779-31fb-4eb4-904b-b18516922b5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#extract threaded posts and quoted posts\n",
    "#convert to embeddings\n",
    "CITY, city_paths, city_chains, city_embeddings, city_singleton_ids,city_singleton_embeddings, city_singleton_texts = make_thread_embeddings(citydata,smodel)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "faf4e079-5149-4bed-8a6a-24806012e0be",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Or load preprocessed City-Data.com Corpus data\n",
    "city = open_pickle('citydata_processed.pkl')\n",
    "\n",
    "CITY = city[0] #networkx graph\n",
    "city_paths = city[1] #post and quoted posts by post_id\n",
    "city_chains = city[2] #treaded texts\n",
    "city_embeddings = city[3] #threaded embeddings\n",
    "city_singleton_ids = city[4] #singleton post_ids\n",
    "city_singleton_embeddings = city[5] #singleton post embeddings\n",
    "city_singleton_texts = city[6] #singleton posts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "e13c274c-99b8-4fe5-a8fc-03819cf71844",
   "metadata": {},
   "outputs": [],
   "source": [
    "threads = city_chains + city_singleton_texts"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f317428-2aa8-4f0a-ac7e-455cc9285198",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Sentence Embedding-Based Topic Modeler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "b0a2e8a1-43af-4cae-9b3c-4966b1ef2e33",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem import WordNetLemmatizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.cluster import KMeans, SpectralClustering, kmeans_plusplus\n",
    "from sklearn.manifold import SpectralEmbedding\n",
    "from nltk.corpus import stopwords\n",
    "import nltk\n",
    "import re\n",
    "from scipy.cluster.hierarchy import fclusterdata\n",
    "from sklearn.datasets import load_digits\n",
    "from sklearn.feature_selection import SelectKBest, chi2\n",
    "\n",
    "def topic_model(texts, transformer, clusters=3,vectorizer='cv',mindf=5, ngrams=(1,1),kbest=5000,init_='k-means++'):\n",
    "    \"\"\"\n",
    "    Function to SE-Topic Model City-Data.com Corpus.\n",
    "\n",
    "    Args:\n",
    "        texts: list of strings\n",
    "        transformer: sentence-transformer model \n",
    "        clusters: (int) number of topics to derive\n",
    "        vectorizer: (string) 'cv' or 'tfidf'\n",
    "        mindf: (int) minimum threshold for token inclusion\n",
    "        ngrams: (tuple) ngram range for tokens\n",
    "        kbest: max textual features for topic modeling\n",
    "        init_: string or list of topic priors\n",
    "    Returns:\n",
    "        dictionary: dataframe of texts, topical term weights, vectroizer, clusterer, topics\n",
    "    \"\"\"\n",
    "        \n",
    "    embeddings = transformer.encode(texts)\n",
    "    #if cluster centers provided:\n",
    "    if type(init_) == np.ndarray:\n",
    "        print('setting cluster centers')\n",
    "    \n",
    "    #instantiate kmeans clusterer \n",
    "    km = KMeans(n_clusters=clusters,init=init_, random_state=0)\n",
    "    km.fit(embeddings)\n",
    "\n",
    "    #process texts for topical word extraction\n",
    "    processed_threads = [text_process(text) for text in texts]\n",
    "\n",
    "    #group text by kmeans cluster label\n",
    "    df = pd.DataFrame({'text':processed_threads,'label':km.labels_})    \n",
    "    df_grouped = df.groupby('label')['text'].apply(list)\n",
    "\n",
    "    if vectorizer == 'cv': #sklearn CountVectorizer\n",
    "        cv = CountVectorizer(stop_words='english',min_df=mindf,ngram_range=ngrams)\n",
    "        CX = cv.fit_transform(df.text)\n",
    "        \n",
    "        #select kbest features\n",
    "        kbc = SelectKBest(chi2, k=kbest).fit(CX, km.labels_)\n",
    "\n",
    "        ff = pd.DataFrame()\n",
    "        ff['term'] = np.asarray(cv.get_feature_names_out())[kbc.get_support()]\n",
    "        #extract topical terms\n",
    "       \n",
    "        for i in range(clusters):\n",
    "            ff['topic_'+str(i)] = kbc.transform(cv.transform([' '.join(df.groupby('label')['text'].apply(list)[i])])).toarray().tolist()[0]\n",
    "                                                 \n",
    "        ff = pd.DataFrame([ff.iloc[i] for i in range(len(ff)) if len(ff.iloc[i]['term']) > 3])\n",
    "        topics = get_topical_terms(ff,term_count=25)\n",
    "        return {'texts':df,'term_weights':ff,'vectorizer':cv, 'clusterer':km, 'topics':topics}\n",
    "   \n",
    "    elif vectorizer == 'tfidf':\n",
    "        tfidf = TfidfVectorizer(stop_words='english',min_df=mindf, ngram_range=ngrams)\n",
    "        CX = tfidf.fit_transform(df.text)\n",
    "        kbc = SelectKBest(chi2, k=kbest).fit(CX, km.labels_)\n",
    "\n",
    "        ff = pd.DataFrame()\n",
    "        ff['term'] = np.asarray(tfidf.get_feature_names_out())[kbc.get_support()]\n",
    "       \n",
    "        for i in range(clusters):\n",
    "            ff['topic_'+str(i)] = kbc.transform(tfidf.transform([' '.join(df.groupby('label')['text'].apply(list)[i])])).toarray().tolist()[0]\n",
    "   \n",
    "        ff = pd.DataFrame([ff.iloc[i] for i in range(len(ff)) if len(ff.iloc[i]['term']) > 3])\n",
    "        topics = get_topical_terms(ff,term_count=25)\n",
    "       \n",
    "\n",
    "        return {'texts':df,'term_weights':ff,'vectorizer':tfidf, 'clusterer':km, 'topics':topics}\n",
    "\n",
    "def get_topical_terms(dataframe, term_count=25):\n",
    "    topics = []\n",
    "    for i in range(len(dataframe.columns)-1):\n",
    "        \n",
    "        terms = dataframe.sort_values(by='coef_'+str(i),ascending=False)[:term_count]['term'].tolist()\n",
    "        topics.append(terms)\n",
    "    filtered = []\n",
    "    for topic in topics:\n",
    "        t = []\n",
    "        for term in topic:\n",
    "            if len(term.split()) > 1:\n",
    "                t.append('_'.join(term.split()))\n",
    "            else:\n",
    "                t.append(term)\n",
    "        filtered.append(t)\n",
    "    return filtered\n",
    "import numpy as np\n",
    "\n",
    "from sentence_transformers import SentenceTransformer\n",
    "\n",
    "#instantiate transformer model to create embeddings\n",
    "smodel = SentenceTransformer('multi-qa-MiniLM-L6-cos-v1')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efed305d-965e-44eb-b315-09e32935c168",
   "metadata": {},
   "source": [
    "### Default SE-Topics Modeling Method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "b3581fed-5a26-4124-ad1c-1980e2c875e3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\rmomi\\anaconda3\\envs\\ktransformers\\Lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning:\n",
      "\n",
      "The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#SE-Topic Model City-Data.com Corpus Threads\n",
    "\n",
    "se_thread_topics = topic_model(threads, smodel, clusters=4,vectorizer='cv',mindf=10, ngrams=(1,3),kbest=7500,init_='k-means++')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "662ed013-4df4-4e53-8f82-2322bff3db13",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "      <th>21</th>\n",
       "      <th>22</th>\n",
       "      <th>23</th>\n",
       "      <th>24</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>city</td>\n",
       "      <td>philadelphia</td>\n",
       "      <td>philly</td>\n",
       "      <td>people</td>\n",
       "      <td>think</td>\n",
       "      <td>street</td>\n",
       "      <td>year</td>\n",
       "      <td>area</td>\n",
       "      <td>center</td>\n",
       "      <td>building</td>\n",
       "      <td>...</td>\n",
       "      <td>need</td>\n",
       "      <td>make</td>\n",
       "      <td>really</td>\n",
       "      <td>place</td>\n",
       "      <td>going</td>\n",
       "      <td>want</td>\n",
       "      <td>retail</td>\n",
       "      <td>development</td>\n",
       "      <td>better</td>\n",
       "      <td>market</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>store</td>\n",
       "      <td>city</td>\n",
       "      <td>building</td>\n",
       "      <td>market</td>\n",
       "      <td>street</td>\n",
       "      <td>think</td>\n",
       "      <td>retail</td>\n",
       "      <td>center</td>\n",
       "      <td>walnut</td>\n",
       "      <td>going</td>\n",
       "      <td>...</td>\n",
       "      <td>look</td>\n",
       "      <td>really</td>\n",
       "      <td>project</td>\n",
       "      <td>time</td>\n",
       "      <td>good</td>\n",
       "      <td>center_city</td>\n",
       "      <td>philadelphia</td>\n",
       "      <td>chestnut</td>\n",
       "      <td>location</td>\n",
       "      <td>retailer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>good</td>\n",
       "      <td>finally</td>\n",
       "      <td>abandon</td>\n",
       "      <td>politics</td>\n",
       "      <td>poorest_neighborhood</td>\n",
       "      <td>poorest</td>\n",
       "      <td>poorer</td>\n",
       "      <td>poor_white</td>\n",
       "      <td>poor_people</td>\n",
       "      <td>poor</td>\n",
       "      <td>...</td>\n",
       "      <td>politician</td>\n",
       "      <td>poorly_executed</td>\n",
       "      <td>politically</td>\n",
       "      <td>political_landscape</td>\n",
       "      <td>political_culture</td>\n",
       "      <td>political</td>\n",
       "      <td>politely</td>\n",
       "      <td>polite</td>\n",
       "      <td>policy</td>\n",
       "      <td>policing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>people</td>\n",
       "      <td>city</td>\n",
       "      <td>crime</td>\n",
       "      <td>time</td>\n",
       "      <td>year</td>\n",
       "      <td>white</td>\n",
       "      <td>think</td>\n",
       "      <td>philly</td>\n",
       "      <td>going</td>\n",
       "      <td>black</td>\n",
       "      <td>...</td>\n",
       "      <td>good</td>\n",
       "      <td>need</td>\n",
       "      <td>police</td>\n",
       "      <td>want</td>\n",
       "      <td>news</td>\n",
       "      <td>area</td>\n",
       "      <td>philadelphia</td>\n",
       "      <td>thing</td>\n",
       "      <td>life</td>\n",
       "      <td>sure</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       0             1         2         3                     4        5   \\\n",
       "0    city  philadelphia    philly    people                 think   street   \n",
       "1   store          city  building    market                street    think   \n",
       "2    good       finally   abandon  politics  poorest_neighborhood  poorest   \n",
       "3  people          city     crime      time                  year    white   \n",
       "\n",
       "       6           7            8         9   ...          15  \\\n",
       "0    year        area       center  building  ...        need   \n",
       "1  retail      center       walnut     going  ...        look   \n",
       "2  poorer  poor_white  poor_people      poor  ...  politician   \n",
       "3   think      philly        going     black  ...        good   \n",
       "\n",
       "                16           17                   18                 19  \\\n",
       "0             make       really                place              going   \n",
       "1           really      project                 time               good   \n",
       "2  poorly_executed  politically  political_landscape  political_culture   \n",
       "3             need       police                 want               news   \n",
       "\n",
       "            20            21           22        23        24  \n",
       "0         want        retail  development    better    market  \n",
       "1  center_city  philadelphia     chestnut  location  retailer  \n",
       "2    political      politely       polite    policy  policing  \n",
       "3         area  philadelphia        thing      life      sure  \n",
       "\n",
       "[4 rows x 25 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#SE-Topic Model City-Data.com Corpus Threads with top-25 terms\n",
    "pd.DataFrame(se_thread_topics['topics'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "082c8c8a-d81f-41b0-9d4a-1eda62b34d78",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\rmomi\\anaconda3\\envs\\ktransformers\\Lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning:\n",
      "\n",
      "The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#SE-Topic Model City-Data.com Corpus Posts\n",
    "se_post_topics = topic_model(posts, smodel, clusters=4,vectorizer='cv',mindf=10, ngrams=(1,3),kbest=7500,init_='k-means++')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "d4885d03-7424-47e5-b37d-7704772aca89",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "      <th>21</th>\n",
       "      <th>22</th>\n",
       "      <th>23</th>\n",
       "      <th>24</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>people</td>\n",
       "      <td>crime</td>\n",
       "      <td>city</td>\n",
       "      <td>think</td>\n",
       "      <td>white</td>\n",
       "      <td>year</td>\n",
       "      <td>time</td>\n",
       "      <td>going</td>\n",
       "      <td>black</td>\n",
       "      <td>make</td>\n",
       "      <td>...</td>\n",
       "      <td>street</td>\n",
       "      <td>need</td>\n",
       "      <td>police</td>\n",
       "      <td>want</td>\n",
       "      <td>news</td>\n",
       "      <td>life</td>\n",
       "      <td>area</td>\n",
       "      <td>thing</td>\n",
       "      <td>look</td>\n",
       "      <td>sure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>store</td>\n",
       "      <td>city</td>\n",
       "      <td>building</td>\n",
       "      <td>street</td>\n",
       "      <td>think</td>\n",
       "      <td>market</td>\n",
       "      <td>center</td>\n",
       "      <td>retail</td>\n",
       "      <td>people</td>\n",
       "      <td>going</td>\n",
       "      <td>...</td>\n",
       "      <td>center_city</td>\n",
       "      <td>time</td>\n",
       "      <td>really</td>\n",
       "      <td>project</td>\n",
       "      <td>good</td>\n",
       "      <td>look</td>\n",
       "      <td>philadelphia</td>\n",
       "      <td>chestnut</td>\n",
       "      <td>location</td>\n",
       "      <td>retailer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>abandoned</td>\n",
       "      <td>plain</td>\n",
       "      <td>planted</td>\n",
       "      <td>plant</td>\n",
       "      <td>planning_commission</td>\n",
       "      <td>planning</td>\n",
       "      <td>planner</td>\n",
       "      <td>planned</td>\n",
       "      <td>plan_really</td>\n",
       "      <td>plan_philly</td>\n",
       "      <td>...</td>\n",
       "      <td>preserved</td>\n",
       "      <td>place_taking</td>\n",
       "      <td>place_start</td>\n",
       "      <td>place_philly</td>\n",
       "      <td>place_philadelphia</td>\n",
       "      <td>place_people</td>\n",
       "      <td>place_outside</td>\n",
       "      <td>place_live</td>\n",
       "      <td>place_food_court</td>\n",
       "      <td>place_food</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>city</td>\n",
       "      <td>philadelphia</td>\n",
       "      <td>philly</td>\n",
       "      <td>people</td>\n",
       "      <td>think</td>\n",
       "      <td>street</td>\n",
       "      <td>year</td>\n",
       "      <td>area</td>\n",
       "      <td>center</td>\n",
       "      <td>neighborhood</td>\n",
       "      <td>...</td>\n",
       "      <td>make</td>\n",
       "      <td>really</td>\n",
       "      <td>need</td>\n",
       "      <td>place</td>\n",
       "      <td>crime</td>\n",
       "      <td>want</td>\n",
       "      <td>going</td>\n",
       "      <td>development</td>\n",
       "      <td>retail</td>\n",
       "      <td>better</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0             1         2       3                    4         5   \\\n",
       "0     people         crime      city   think                white      year   \n",
       "1      store          city  building  street                think    market   \n",
       "2  abandoned         plain   planted   plant  planning_commission  planning   \n",
       "3       city  philadelphia    philly  people                think    street   \n",
       "\n",
       "        6        7            8             9   ...           15  \\\n",
       "0     time    going        black          make  ...       street   \n",
       "1   center   retail       people         going  ...  center_city   \n",
       "2  planner  planned  plan_really   plan_philly  ...    preserved   \n",
       "3     year     area       center  neighborhood  ...         make   \n",
       "\n",
       "             16           17            18                  19            20  \\\n",
       "0          need       police          want                news          life   \n",
       "1          time       really       project                good          look   \n",
       "2  place_taking  place_start  place_philly  place_philadelphia  place_people   \n",
       "3        really         need         place               crime          want   \n",
       "\n",
       "              21           22                23          24  \n",
       "0           area        thing              look        sure  \n",
       "1   philadelphia     chestnut          location    retailer  \n",
       "2  place_outside   place_live  place_food_court  place_food  \n",
       "3          going  development            retail      better  \n",
       "\n",
       "[4 rows x 25 columns]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#SE-Topic Model City-Data.com Corpus Posts with top-25 terms\n",
    "pd.DataFrame(se_post_topics['topics'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "21dfffd9-ee46-44bd-ad77-9697c61b5ac6",
   "metadata": {},
   "outputs": [],
   "source": [
    "citydata.fillna('',inplace=True)\n",
    "coronavirus = pd.DataFrame([citydata.iloc[i] for i in range(len(citydata)) if citydata.iloc[i]['forum'] == 'coronavirus'])\n",
    "crime = pd.DataFrame([citydata.iloc[i] for i in range(len(citydata)) if citydata.iloc[i]['forum'] == 'crime'])\n",
    "metro = pd.DataFrame([citydata.iloc[i] for i in range(len(citydata)) if citydata.iloc[i]['forum'] == 'metro'])\n",
    "plan = pd.DataFrame([citydata.iloc[i] for i in range(len(citydata)) if citydata.iloc[i]['forum'] == 'plan'])\n",
    "retail = pd.DataFrame([citydata.iloc[i] for i in range(len(citydata)) if citydata.iloc[i]['forum'] == 'retail'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af040f7f-b280-4240-81f9-680c30621c46",
   "metadata": {},
   "source": [
    "### Guided SE-Topics Post (High Degree Posts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "b32483c0-6b2f-47db-9088-9261f0d6eaf9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_high_degree_nodes(graph):\n",
    "    high_degree_node_text =[]\n",
    "    for (k,v) in sorted(dict(graph.degree()).items(), key=lambda x:x[1],reverse=True): \n",
    "        if graph.nodes()[k]['text'] != None:\n",
    "            high_degree_node_text.append(graph.nodes()[k]['text'])\n",
    "    return high_degree_node_text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "824f0096-8a82-4a46-b867-64410a1f800f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "setting cluster centers\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\rmomi\\anaconda3\\envs\\ktransformers\\Lib\\site-packages\\sklearn\\cluster\\_kmeans.py:870: FutureWarning:\n",
      "\n",
      "The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n",
      "\n",
      "C:\\Users\\rmomi\\anaconda3\\envs\\ktransformers\\Lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1362: RuntimeWarning:\n",
      "\n",
      "Explicit initial center position passed: performing only one init in KMeans instead of n_init=10.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#make networkx graph of forum data\n",
    "CO = make_graph_citydata(coronavirus)\n",
    "CR = make_graph_citydata(crime)\n",
    "ME = make_graph_citydata(metro)\n",
    "PL = make_graph_citydata(plan)\n",
    "R = make_graph_citydata(retail)\n",
    "\n",
    "#extract high degree posts/nodes\n",
    "coronavirus_nodes = extract_high_degree_nodes(CO)[0]\n",
    "crime_nodes = extract_high_degree_nodes(CR)[0]\n",
    "metro_nodes = extract_high_degree_nodes(ME)[0]\n",
    "plan_nodes = extract_high_degree_nodes(PL)[0]\n",
    "retail_nodes = extract_high_degree_nodes(R)[0]\n",
    "\n",
    "#convert high degree posts into sentence embeddings\n",
    "degree_seed_embeddings = smodel.encode([coronavirus_nodes,crime_nodes + metro_nodes, plan_nodes, retail_nodes])\n",
    "\n",
    "se_post_degree_topics = topic_model(posts, smodel, clusters=4,vectorizer='cv',mindf=10, ngrams=(1,3),kbest=7500,init_=degree_seed_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "id": "e3a4c83f-f644-4ded-83fa-7d41f9805fa4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>people</td>\n",
       "      <td>think</td>\n",
       "      <td>year</td>\n",
       "      <td>time</td>\n",
       "      <td>going</td>\n",
       "      <td>good</td>\n",
       "      <td>look</td>\n",
       "      <td>really</td>\n",
       "      <td>project</td>\n",
       "      <td>project</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>city</td>\n",
       "      <td>people</td>\n",
       "      <td>crime</td>\n",
       "      <td>philly</td>\n",
       "      <td>year</td>\n",
       "      <td>street</td>\n",
       "      <td>think</td>\n",
       "      <td>time</td>\n",
       "      <td>white</td>\n",
       "      <td>white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>city</td>\n",
       "      <td>store</td>\n",
       "      <td>philadelphia</td>\n",
       "      <td>think</td>\n",
       "      <td>building</td>\n",
       "      <td>philly</td>\n",
       "      <td>people</td>\n",
       "      <td>street</td>\n",
       "      <td>center</td>\n",
       "      <td>center</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>abandon</td>\n",
       "      <td>powelton</td>\n",
       "      <td>prediction</td>\n",
       "      <td>preceding</td>\n",
       "      <td>precaution</td>\n",
       "      <td>praying_rosary</td>\n",
       "      <td>praying</td>\n",
       "      <td>prank</td>\n",
       "      <td>prada</td>\n",
       "      <td>prada</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         0         1             2          3           4               5  \\\n",
       "0   people     think          year       time       going            good   \n",
       "1     city    people         crime     philly        year          street   \n",
       "2     city     store  philadelphia      think    building          philly   \n",
       "3  abandon  powelton    prediction  preceding  precaution  praying_rosary   \n",
       "\n",
       "         6       7        8        8  \n",
       "0     look  really  project  project  \n",
       "1    think    time    white    white  \n",
       "2   people  street   center   center  \n",
       "3  praying   prank    prada    prada  "
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(se_post_degree_topics['topics'])[[0,1,2,3,4,5,6,7,8,8]]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a9f9b4a-d5f5-4b06-8f6e-4c1d441fd93b",
   "metadata": {},
   "source": [
    "### Guided SE-Topics Threads (High Degree Posts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ea4be0b-be0d-4603-a267-082fb79745fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "se_threads_degree_topics = topic_model(threads, smodel, clusters=4,vectorizer='cv',mindf=10, ngrams=(1,3),kbest=7500,init_=degree_seed_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2796d9d6-be9b-4f15-8795-d5826d215ff8",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(se_threads_degree_topics['topics'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "129b79e6-f896-4831-be48-2f2e155c7822",
   "metadata": {},
   "source": [
    "### Guided SE-Topics Posts (Initial Posts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "e68a723d-b957-4e9b-9887-050bd31db5bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Extract initial posts from each forum\n",
    "first_posts =[\n",
    "    coronavirus.post.tolist()[0],\n",
    "    crime.post.tolist()[0] +\n",
    "    metro.post.tolist()[0],\n",
    "    plan.post.tolist()[0],\n",
    "    retail.post.tolist()[0]\n",
    "]\n",
    "\n",
    "#Convert posts to sentence-embeddings\n",
    "first_posts_embeddings = smodel.encode(first_posts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77c66892-9f45-4488-bc81-7919db3806aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "se_posts_init_posts_topics = topic_model(posts, smodel, clusters=4,vectorizer='cv',mindf=10, ngrams=(1,3),kbest=7500,init_=first_posts_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "id": "2b71fb2f-43e3-4d9d-a41e-9ce81c388626",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "      <th>21</th>\n",
       "      <th>22</th>\n",
       "      <th>23</th>\n",
       "      <th>24</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>people</td>\n",
       "      <td>crime</td>\n",
       "      <td>city</td>\n",
       "      <td>white</td>\n",
       "      <td>think</td>\n",
       "      <td>year</td>\n",
       "      <td>time</td>\n",
       "      <td>going</td>\n",
       "      <td>black</td>\n",
       "      <td>make</td>\n",
       "      <td>...</td>\n",
       "      <td>good</td>\n",
       "      <td>need</td>\n",
       "      <td>police</td>\n",
       "      <td>want</td>\n",
       "      <td>news</td>\n",
       "      <td>life</td>\n",
       "      <td>thing</td>\n",
       "      <td>area</td>\n",
       "      <td>actually</td>\n",
       "      <td>look</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>city</td>\n",
       "      <td>philly</td>\n",
       "      <td>philadelphia</td>\n",
       "      <td>people</td>\n",
       "      <td>think</td>\n",
       "      <td>street</td>\n",
       "      <td>year</td>\n",
       "      <td>area</td>\n",
       "      <td>center</td>\n",
       "      <td>time</td>\n",
       "      <td>...</td>\n",
       "      <td>really</td>\n",
       "      <td>make</td>\n",
       "      <td>place</td>\n",
       "      <td>store</td>\n",
       "      <td>want</td>\n",
       "      <td>building</td>\n",
       "      <td>going</td>\n",
       "      <td>north</td>\n",
       "      <td>better</td>\n",
       "      <td>retail</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>city</td>\n",
       "      <td>building</td>\n",
       "      <td>street</td>\n",
       "      <td>think</td>\n",
       "      <td>project</td>\n",
       "      <td>people</td>\n",
       "      <td>center</td>\n",
       "      <td>market</td>\n",
       "      <td>tower</td>\n",
       "      <td>going</td>\n",
       "      <td>...</td>\n",
       "      <td>line</td>\n",
       "      <td>really</td>\n",
       "      <td>time</td>\n",
       "      <td>space</td>\n",
       "      <td>parking</td>\n",
       "      <td>make</td>\n",
       "      <td>plan</td>\n",
       "      <td>good</td>\n",
       "      <td>center_city</td>\n",
       "      <td>need</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>store</td>\n",
       "      <td>city</td>\n",
       "      <td>retail</td>\n",
       "      <td>market</td>\n",
       "      <td>think</td>\n",
       "      <td>mall</td>\n",
       "      <td>center</td>\n",
       "      <td>retailer</td>\n",
       "      <td>walnut</td>\n",
       "      <td>philadelphia</td>\n",
       "      <td>...</td>\n",
       "      <td>chestnut</td>\n",
       "      <td>year</td>\n",
       "      <td>shopping</td>\n",
       "      <td>going</td>\n",
       "      <td>area</td>\n",
       "      <td>time</td>\n",
       "      <td>brand</td>\n",
       "      <td>shop</td>\n",
       "      <td>really</td>\n",
       "      <td>space</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       0         1             2       3        4       5       6         7   \\\n",
       "0  people     crime          city   white    think    year    time     going   \n",
       "1    city    philly  philadelphia  people    think  street    year      area   \n",
       "2    city  building        street   think  project  people  center    market   \n",
       "3   store      city        retail  market    think    mall  center  retailer   \n",
       "\n",
       "       8             9   ...        15      16        17     18       19  \\\n",
       "0   black          make  ...      good    need    police   want     news   \n",
       "1  center          time  ...    really    make     place  store     want   \n",
       "2   tower         going  ...      line  really      time  space  parking   \n",
       "3  walnut  philadelphia  ...  chestnut    year  shopping  going     area   \n",
       "\n",
       "         20     21     22           23      24  \n",
       "0      life  thing   area     actually    look  \n",
       "1  building  going  north       better  retail  \n",
       "2      make   plan   good  center_city    need  \n",
       "3      time  brand   shop       really   space  \n",
       "\n",
       "[4 rows x 25 columns]"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(se_posts_init_posts_topics['topics'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad8d2b8b-8f2b-425f-97df-72689b134d83",
   "metadata": {},
   "source": [
    "### Guided SE-Topics Threads (Initial Posts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa09c0ec-b7b0-40e1-a8d5-38758e1e5136",
   "metadata": {},
   "outputs": [],
   "source": [
    "se_threads_init_posts_topics = topic_model(threads, smodel, clusters=4,vectorizer='cv',mindf=10, ngrams=(1,3),kbest=7500,init_=first_posts_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "934d96d5-92ac-42d6-a6b8-4f3acd8645c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "      <th>21</th>\n",
       "      <th>22</th>\n",
       "      <th>23</th>\n",
       "      <th>24</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>people</td>\n",
       "      <td>city</td>\n",
       "      <td>crime</td>\n",
       "      <td>time</td>\n",
       "      <td>year</td>\n",
       "      <td>white</td>\n",
       "      <td>think</td>\n",
       "      <td>philly</td>\n",
       "      <td>going</td>\n",
       "      <td>black</td>\n",
       "      <td>...</td>\n",
       "      <td>good</td>\n",
       "      <td>need</td>\n",
       "      <td>police</td>\n",
       "      <td>want</td>\n",
       "      <td>news</td>\n",
       "      <td>area</td>\n",
       "      <td>philadelphia</td>\n",
       "      <td>thing</td>\n",
       "      <td>life</td>\n",
       "      <td>sure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>city</td>\n",
       "      <td>philadelphia</td>\n",
       "      <td>philly</td>\n",
       "      <td>people</td>\n",
       "      <td>think</td>\n",
       "      <td>street</td>\n",
       "      <td>year</td>\n",
       "      <td>area</td>\n",
       "      <td>center</td>\n",
       "      <td>building</td>\n",
       "      <td>...</td>\n",
       "      <td>need</td>\n",
       "      <td>make</td>\n",
       "      <td>really</td>\n",
       "      <td>place</td>\n",
       "      <td>going</td>\n",
       "      <td>want</td>\n",
       "      <td>retail</td>\n",
       "      <td>development</td>\n",
       "      <td>better</td>\n",
       "      <td>market</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>store</td>\n",
       "      <td>city</td>\n",
       "      <td>building</td>\n",
       "      <td>market</td>\n",
       "      <td>street</td>\n",
       "      <td>think</td>\n",
       "      <td>retail</td>\n",
       "      <td>center</td>\n",
       "      <td>walnut</td>\n",
       "      <td>going</td>\n",
       "      <td>...</td>\n",
       "      <td>look</td>\n",
       "      <td>really</td>\n",
       "      <td>project</td>\n",
       "      <td>time</td>\n",
       "      <td>good</td>\n",
       "      <td>center_city</td>\n",
       "      <td>philadelphia</td>\n",
       "      <td>chestnut</td>\n",
       "      <td>location</td>\n",
       "      <td>retailer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>good</td>\n",
       "      <td>finally</td>\n",
       "      <td>abandon</td>\n",
       "      <td>politics</td>\n",
       "      <td>poorest_neighborhood</td>\n",
       "      <td>poorest</td>\n",
       "      <td>poorer</td>\n",
       "      <td>poor_white</td>\n",
       "      <td>poor_people</td>\n",
       "      <td>poor</td>\n",
       "      <td>...</td>\n",
       "      <td>politician</td>\n",
       "      <td>poorly_executed</td>\n",
       "      <td>politically</td>\n",
       "      <td>political_landscape</td>\n",
       "      <td>political_culture</td>\n",
       "      <td>political</td>\n",
       "      <td>politely</td>\n",
       "      <td>polite</td>\n",
       "      <td>policy</td>\n",
       "      <td>policing</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       0             1         2         3                     4        5   \\\n",
       "0  people          city     crime      time                  year    white   \n",
       "1    city  philadelphia    philly    people                 think   street   \n",
       "2   store          city  building    market                street    think   \n",
       "3    good       finally   abandon  politics  poorest_neighborhood  poorest   \n",
       "\n",
       "       6           7            8         9   ...          15  \\\n",
       "0   think      philly        going     black  ...        good   \n",
       "1    year        area       center  building  ...        need   \n",
       "2  retail      center       walnut     going  ...        look   \n",
       "3  poorer  poor_white  poor_people      poor  ...  politician   \n",
       "\n",
       "                16           17                   18                 19  \\\n",
       "0             need       police                 want               news   \n",
       "1             make       really                place              going   \n",
       "2           really      project                 time               good   \n",
       "3  poorly_executed  politically  political_landscape  political_culture   \n",
       "\n",
       "            20            21           22        23        24  \n",
       "0         area  philadelphia        thing      life      sure  \n",
       "1         want        retail  development    better    market  \n",
       "2  center_city  philadelphia     chestnut  location  retailer  \n",
       "3    political      politely       polite    policy  policing  \n",
       "\n",
       "[4 rows x 25 columns]"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(se_threads_init_posts_topics['topics'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c4b447dd-e00b-4374-aeee-55ca095f18e0",
   "metadata": {},
   "source": [
    "## Guided SE-Topics Posts (Forum Titles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "e2976e37-8b1f-4b53-991b-ffc32da0dec6",
   "metadata": {},
   "outputs": [],
   "source": [
    "titles= \"\"\"How's everyone doing amongst the Coronavirus shut down?\n",
    "Official Greater Philadelphia Area Crime Thread (Chester, New Castle: 2013, middle school, university) Official Philadelphia Metro Crime Thread (York, Chester: apartment complexes, houses, unemployment) \n",
    "Philadelphia 2035 (Houston: foreclosure, neighborhoods, wage) \n",
    "Retail coming to Philadelphia (Penn, Burlington: real estate, house, buying)\"\"\"\n",
    "\n",
    "title_embeddings = smodel.encode(titles.split('\\n'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e660c80-3540-44cc-918a-df0b80a8796b",
   "metadata": {},
   "outputs": [],
   "source": [
    "se_posts_title_topics = topic_model(posts, smodel, clusters=4,vectorizer='cv',mindf=10, ngrams=(1,3),kbest=7500,init_=title_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "71463168-da2e-4fea-b6ef-b46def8d7a07",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "      <th>21</th>\n",
       "      <th>22</th>\n",
       "      <th>23</th>\n",
       "      <th>24</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>people</td>\n",
       "      <td>think</td>\n",
       "      <td>year</td>\n",
       "      <td>time</td>\n",
       "      <td>going</td>\n",
       "      <td>good</td>\n",
       "      <td>really</td>\n",
       "      <td>right</td>\n",
       "      <td>look</td>\n",
       "      <td>project</td>\n",
       "      <td>...</td>\n",
       "      <td>actually</td>\n",
       "      <td>article</td>\n",
       "      <td>want</td>\n",
       "      <td>news</td>\n",
       "      <td>sure</td>\n",
       "      <td>probably</td>\n",
       "      <td>thing</td>\n",
       "      <td>long</td>\n",
       "      <td>philly</td>\n",
       "      <td>hope</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>people</td>\n",
       "      <td>crime</td>\n",
       "      <td>city</td>\n",
       "      <td>white</td>\n",
       "      <td>year</td>\n",
       "      <td>philly</td>\n",
       "      <td>murder</td>\n",
       "      <td>time</td>\n",
       "      <td>think</td>\n",
       "      <td>black</td>\n",
       "      <td>...</td>\n",
       "      <td>right</td>\n",
       "      <td>really</td>\n",
       "      <td>drug</td>\n",
       "      <td>need</td>\n",
       "      <td>area</td>\n",
       "      <td>good</td>\n",
       "      <td>neighborhood</td>\n",
       "      <td>violent</td>\n",
       "      <td>case</td>\n",
       "      <td>shooting</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>city</td>\n",
       "      <td>philadelphia</td>\n",
       "      <td>philly</td>\n",
       "      <td>people</td>\n",
       "      <td>street</td>\n",
       "      <td>building</td>\n",
       "      <td>think</td>\n",
       "      <td>area</td>\n",
       "      <td>year</td>\n",
       "      <td>center</td>\n",
       "      <td>...</td>\n",
       "      <td>need</td>\n",
       "      <td>really</td>\n",
       "      <td>going</td>\n",
       "      <td>make</td>\n",
       "      <td>center_city</td>\n",
       "      <td>place</td>\n",
       "      <td>want</td>\n",
       "      <td>look</td>\n",
       "      <td>line</td>\n",
       "      <td>tower</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>store</td>\n",
       "      <td>city</td>\n",
       "      <td>retail</td>\n",
       "      <td>market</td>\n",
       "      <td>center</td>\n",
       "      <td>think</td>\n",
       "      <td>street</td>\n",
       "      <td>mall</td>\n",
       "      <td>philadelphia</td>\n",
       "      <td>walnut</td>\n",
       "      <td>...</td>\n",
       "      <td>building</td>\n",
       "      <td>area</td>\n",
       "      <td>chestnut</td>\n",
       "      <td>space</td>\n",
       "      <td>philly</td>\n",
       "      <td>going</td>\n",
       "      <td>year</td>\n",
       "      <td>good</td>\n",
       "      <td>time</td>\n",
       "      <td>really</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       0             1       2       3       4         5       6      7   \\\n",
       "0  people         think    year    time   going      good  really  right   \n",
       "1  people         crime    city   white    year    philly  murder   time   \n",
       "2    city  philadelphia  philly  people  street  building   think   area   \n",
       "3   store          city  retail  market  center     think  street   mall   \n",
       "\n",
       "             8        9   ...        15       16        17     18  \\\n",
       "0          look  project  ...  actually  article      want   news   \n",
       "1         think    black  ...     right   really      drug   need   \n",
       "2          year   center  ...      need   really     going   make   \n",
       "3  philadelphia   walnut  ...  building     area  chestnut  space   \n",
       "\n",
       "            19        20            21       22      23        24  \n",
       "0         sure  probably         thing     long  philly      hope  \n",
       "1         area      good  neighborhood  violent    case  shooting  \n",
       "2  center_city     place          want     look    line     tower  \n",
       "3       philly     going          year     good    time    really  \n",
       "\n",
       "[4 rows x 25 columns]"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(se_posts_title_topics['topics'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "715ef4b6-7fc0-4e89-b847-94b03afa51a7",
   "metadata": {},
   "source": [
    "## Guided SE-Topics Threads (Forum Titles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cffb8721-a7aa-444a-b888-c21b8f896d9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "se_threads_title_topics = topic_model(threads, smodel, clusters=4,vectorizer='cv',mindf=10, ngrams=(1,3),kbest=7500,init_=title_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "9b26bd8f-ebbd-4429-9257-b89dbe32e821",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "      <th>21</th>\n",
       "      <th>22</th>\n",
       "      <th>23</th>\n",
       "      <th>24</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>people</td>\n",
       "      <td>city</td>\n",
       "      <td>crime</td>\n",
       "      <td>year</td>\n",
       "      <td>time</td>\n",
       "      <td>think</td>\n",
       "      <td>white</td>\n",
       "      <td>philly</td>\n",
       "      <td>going</td>\n",
       "      <td>black</td>\n",
       "      <td>...</td>\n",
       "      <td>good</td>\n",
       "      <td>need</td>\n",
       "      <td>philadelphia</td>\n",
       "      <td>police</td>\n",
       "      <td>want</td>\n",
       "      <td>area</td>\n",
       "      <td>news</td>\n",
       "      <td>life</td>\n",
       "      <td>look</td>\n",
       "      <td>thing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>finally</td>\n",
       "      <td>abandon</td>\n",
       "      <td>pottery</td>\n",
       "      <td>powerhouse</td>\n",
       "      <td>power</td>\n",
       "      <td>powelton_village</td>\n",
       "      <td>powelton</td>\n",
       "      <td>poverty_rate</td>\n",
       "      <td>poverty_problem</td>\n",
       "      <td>poverty_crime</td>\n",
       "      <td>...</td>\n",
       "      <td>potentially</td>\n",
       "      <td>prestige</td>\n",
       "      <td>potential_customer</td>\n",
       "      <td>potential_city</td>\n",
       "      <td>potential</td>\n",
       "      <td>posting</td>\n",
       "      <td>poster</td>\n",
       "      <td>post</td>\n",
       "      <td>possibly</td>\n",
       "      <td>possible</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>city</td>\n",
       "      <td>philadelphia</td>\n",
       "      <td>philly</td>\n",
       "      <td>people</td>\n",
       "      <td>building</td>\n",
       "      <td>street</td>\n",
       "      <td>think</td>\n",
       "      <td>year</td>\n",
       "      <td>area</td>\n",
       "      <td>center</td>\n",
       "      <td>...</td>\n",
       "      <td>need</td>\n",
       "      <td>really</td>\n",
       "      <td>going</td>\n",
       "      <td>make</td>\n",
       "      <td>center_city</td>\n",
       "      <td>look</td>\n",
       "      <td>place</td>\n",
       "      <td>want</td>\n",
       "      <td>tower</td>\n",
       "      <td>line</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>store</td>\n",
       "      <td>city</td>\n",
       "      <td>retail</td>\n",
       "      <td>market</td>\n",
       "      <td>think</td>\n",
       "      <td>center</td>\n",
       "      <td>street</td>\n",
       "      <td>mall</td>\n",
       "      <td>philadelphia</td>\n",
       "      <td>walnut</td>\n",
       "      <td>...</td>\n",
       "      <td>chestnut</td>\n",
       "      <td>going</td>\n",
       "      <td>space</td>\n",
       "      <td>shopping</td>\n",
       "      <td>year</td>\n",
       "      <td>area</td>\n",
       "      <td>good</td>\n",
       "      <td>philly</td>\n",
       "      <td>really</td>\n",
       "      <td>time</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        0             1        2           3         4                 5   \\\n",
       "0   people          city    crime        year      time             think   \n",
       "1  finally       abandon  pottery  powerhouse     power  powelton_village   \n",
       "2     city  philadelphia   philly      people  building            street   \n",
       "3    store          city   retail      market     think            center   \n",
       "\n",
       "         6             7                8              9   ...           15  \\\n",
       "0     white        philly            going          black  ...         good   \n",
       "1  powelton  poverty_rate  poverty_problem  poverty_crime  ...  potentially   \n",
       "2     think          year             area         center  ...         need   \n",
       "3    street          mall     philadelphia         walnut  ...     chestnut   \n",
       "\n",
       "         16                  17              18           19       20      21  \\\n",
       "0      need        philadelphia          police         want     area    news   \n",
       "1  prestige  potential_customer  potential_city    potential  posting  poster   \n",
       "2    really               going            make  center_city     look   place   \n",
       "3     going               space        shopping         year     area    good   \n",
       "\n",
       "       22        23        24  \n",
       "0    life      look     thing  \n",
       "1    post  possibly  possible  \n",
       "2    want     tower      line  \n",
       "3  philly    really      time  \n",
       "\n",
       "[4 rows x 25 columns]"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(se_threads_title_topics['topics'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7b4908f-d375-4c11-b813-4270beae4c61",
   "metadata": {},
   "source": [
    "## SE-Topics Coherence and Diversity Scores\n",
    "\n",
    "Note: topical diversity scores require: [FastText](https://fasttext.cc/docs/en/english-vectors.html).\n",
    "\n",
    "- Download a model\n",
    "- Load model with gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "7e82aace-bac9-4f07-aa40-6781ad83c5af",
   "metadata": {},
   "outputs": [],
   "source": [
    "#load fasttext model locally+\n",
    "fast = gensim.models.fasttext.load_facebook_model('crawl-300d-2M-subword/crawl-300d-2M-subword.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "id": "8ffb0599-5114-412a-b3f0-35b8d60fbb69",
   "metadata": {},
   "outputs": [],
   "source": [
    "se_topics_eval_df = pd.concat([\n",
    "        get_coherence_diversity_scores(se_post_topics['texts'].text, se_post_topics['topics'], fast,topn=25,topic_type='posts'),\n",
    "        get_coherence_diversity_scores(se_thread_topics['texts'].text, se_thread_topics['topics'], fast,topn=25,topic_type='threads'),\n",
    "        get_coherence_diversity_scores(se_post_degree_topics['texts'].text, se_threads_degree_topics['topics'], fast,topn=25,topic_type='guided posts (high degree)' ),\n",
    "        get_coherence_diversity_scores(se_threads_degree_topics['texts'].text, se_threads_degree_topics['topics'], fast,topn=25,topic_type='guided threads (high degree)' ),\n",
    "        get_coherence_diversity_scores(se_posts_init_posts_topics['texts'].text, se_posts_init_posts_topics['topics'], fast,topn=25,topic_type='guided posts (initial posts)' ),\n",
    "        get_coherence_diversity_scores(se_threads_init_posts_topics['texts'].text, se_threads_init_posts_topics['topics'], fast,topn=25,topic_type='guided threads (initial posts)' ),\n",
    "        get_coherence_diversity_scores(se_posts_title_topics['texts'].text, se_posts_title_topics['topics'], fast,topn=25,topic_type='guided post (forum titles)' ),\n",
    "        get_coherence_diversity_scores(se_threads_title_topics['texts'].text, se_threads_title_topics['topics'], fast,topn=25,topic_type='guided threads (forum titles)' )])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "6c37a02a-61e9-48c5-aece-ae7ff136b2ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>u_mass_coherence</th>\n",
       "      <th>puw:</th>\n",
       "      <th>jd:</th>\n",
       "      <th>we-cd:</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>posts</th>\n",
       "      <td>-5.382926</td>\n",
       "      <td>0.68</td>\n",
       "      <td>0.803132</td>\n",
       "      <td>0.221566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>threads</th>\n",
       "      <td>-4.572197</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.785500</td>\n",
       "      <td>0.184801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>guided posts (high degree)</th>\n",
       "      <td>-4.627010</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.785500</td>\n",
       "      <td>0.213113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>guided threads (high degree)</th>\n",
       "      <td>-4.534778</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.785500</td>\n",
       "      <td>0.213113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>guided posts (initial posts)</th>\n",
       "      <td>-2.561766</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.620674</td>\n",
       "      <td>0.071358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>guided threads (initial posts)</th>\n",
       "      <td>-4.572197</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.785500</td>\n",
       "      <td>0.218687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>guided post (forum titles)</th>\n",
       "      <td>-2.761955</td>\n",
       "      <td>0.54</td>\n",
       "      <td>0.638784</td>\n",
       "      <td>0.065617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>guided threads (forum titles)</th>\n",
       "      <td>-4.842382</td>\n",
       "      <td>0.69</td>\n",
       "      <td>0.797508</td>\n",
       "      <td>0.065617</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                u_mass_coherence  puw:       jd:    we-cd:\n",
       "posts                                  -5.382926  0.68  0.803132  0.221566\n",
       "threads                                -4.572197  0.67  0.785500  0.184801\n",
       "guided posts (high degree)             -4.627010  0.67  0.785500  0.213113\n",
       "guided threads (high degree)           -4.534778  0.67  0.785500  0.213113\n",
       "guided posts (initial posts)           -2.561766  0.52  0.620674  0.071358\n",
       "guided threads (initial posts)         -4.572197  0.67  0.785500  0.218687\n",
       "guided post (forum titles)             -2.761955  0.54  0.638784  0.065617\n",
       "guided threads (forum titles)          -4.842382  0.69  0.797508  0.065617"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "se_topics_eval_df[['u_mass_coherence','puw:','jd:','we-cd:']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "id": "6e24c623-2034-4255-98b3-3a6fd565db2e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>u_mass_coherence</th>\n",
       "      <th>puw:</th>\n",
       "      <th>jd:</th>\n",
       "      <th>we-cd:</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>-4.231901</td>\n",
       "      <td>0.638750</td>\n",
       "      <td>0.750262</td>\n",
       "      <td>0.156734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.009037</td>\n",
       "      <td>0.067705</td>\n",
       "      <td>0.074847</td>\n",
       "      <td>0.074719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-5.382926</td>\n",
       "      <td>0.520000</td>\n",
       "      <td>0.620674</td>\n",
       "      <td>0.065617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-4.680853</td>\n",
       "      <td>0.637500</td>\n",
       "      <td>0.748821</td>\n",
       "      <td>0.069923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>-4.572197</td>\n",
       "      <td>0.670000</td>\n",
       "      <td>0.785500</td>\n",
       "      <td>0.198957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>-4.091572</td>\n",
       "      <td>0.672500</td>\n",
       "      <td>0.788502</td>\n",
       "      <td>0.214506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>-2.561766</td>\n",
       "      <td>0.690000</td>\n",
       "      <td>0.803132</td>\n",
       "      <td>0.221566</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       u_mass_coherence      puw:       jd:    we-cd:\n",
       "count          8.000000  8.000000  8.000000  8.000000\n",
       "mean          -4.231901  0.638750  0.750262  0.156734\n",
       "std            1.009037  0.067705  0.074847  0.074719\n",
       "min           -5.382926  0.520000  0.620674  0.065617\n",
       "25%           -4.680853  0.637500  0.748821  0.069923\n",
       "50%           -4.572197  0.670000  0.785500  0.198957\n",
       "75%           -4.091572  0.672500  0.788502  0.214506\n",
       "max           -2.561766  0.690000  0.803132  0.221566"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "se_topics_eval_df[['u_mass_coherence','puw:','jd:','we-cd:']].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93f0c02c-1bda-4907-bb71-655409fbec5f",
   "metadata": {},
   "source": [
    "## Gensim Topic Modeling Functions\n",
    "\n",
    "The following functions were used to derive LDA and guided LDA topic models in gensim for comparison:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "id": "703f67b4-8e29-448e-86c4-bcd8c630e454",
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim import models\n",
    "stops = stopwords.words('english') + ['said','know','maybe','post','advertisements','advertisement','posted','thread','like','could','should','would','thing']\n",
    "wn = WordNetLemmatizer()\n",
    "stemmer = nltk.PorterStemmer()\n",
    "#custom text normalizer for City-Data Corpus\n",
    "def text_tokenize(text):\n",
    "    stops = stopwords.words('english') + ['said','know','maybe','post','advertisements','advertisement','posted','thread','like','could','should','would','thing']\n",
    "\n",
    "    text = text.lower()\n",
    "    tokens = nltk.wordpunct_tokenize(text)\n",
    "    tokens = [token for token in tokens if not re.findall(r'\\.com|___|Advertisements|-|_',token)]\n",
    "    lemmas = [wn.lemmatize(token) for token in tokens if len(token) > 2 and token not in stops]\n",
    "    filtered = [lemma for lemma in lemmas if not re.findall(r'[0-9]',lemma) and not re.findall(r'htt',lemma)]\n",
    "    return filtered\n",
    "    \n",
    "def gensim_lda(texts, topic_num=5, topic_word_priors=None,numwords=25, eta_=None, tfidf=False):\n",
    "    \"\"\"gensim lda wrapper with guided topic modeling\"\"\"\n",
    "\n",
    "    if tfidf != False:\n",
    "        #process texts\n",
    "        #build gensim dictionary\n",
    "        processed_texts = [text_tokenize(text) for text in texts]\n",
    "        dictionary = gensim.corpora.Dictionary(processed_texts)\n",
    "    \n",
    "        #build bag-of-words representation\n",
    "        bow = [dictionary.doc2bow(text) for text in processed_texts]\n",
    "        tfidf = models.TfidfModel(bow)\n",
    "        corpus_tfidf = tfidf[bow]\n",
    "        #guided lda with eta\n",
    "        if topic_word_priors and eta_ != None:\n",
    "            etas = []\n",
    "        \n",
    "            for r in range(len(topic_word_priors)):\n",
    "                eta = []\n",
    "                for i in range(len(dictionary)):\n",
    "                    \n",
    "                    if dictionary[i] in topic_word_priors[r]:\n",
    "                        eta.append(np.array(eta_))\n",
    "                    else:\n",
    "                        eta.append(np.array(1/topic_num))\n",
    "                etas.append(eta)\n",
    "    \n",
    "            model = gensim.models.ldamodel.LdaModel(\n",
    "                corpus=corpus_tfidf, id2word=dictionary, num_topics=topic_num,\n",
    "                random_state=42, chunksize=100, eta=np.array(etas),\n",
    "                eval_every=-1, update_every=1,\n",
    "                passes=150, alpha='auto', per_word_topics=True)\n",
    "    \n",
    "            #transform corpus into topics\n",
    "            transformed_corpus = model[corpus_tfidf]\n",
    "    \n",
    "            #extract topical terms\n",
    "            topical_terms = [[token for token in m[1].split('\"') if not re.findall(r'\\d',token)] for m in  model.show_topics(num_words=numwords)]\n",
    "            return model, topical_terms, transformed_corpus, processed_texts\n",
    "            \n",
    "        else:\n",
    "            #standard lda\n",
    "            model = gensim.models.ldamodel.LdaModel(\n",
    "                corpus=corpus_tfidf, id2word=dictionary, num_topics=topic_num,\n",
    "                random_state=42, chunksize=100, eta=None,\n",
    "                eval_every=-1, update_every=1,\n",
    "                passes=150, alpha='auto', per_word_topics=True)\n",
    "            \n",
    "            #transform corpus into topics\n",
    "            transformed_corpus = model[corpus_tfidf]\n",
    "            topical_terms = [[token for token in m[1].split('\"') if not re.findall(r'\\d',token)] for m in  model.show_topics(num_words=numwords)]\n",
    "            return model, topical_terms, transformed_corpus, processed_texts\n",
    "    else:\n",
    "        #process texts\n",
    "        #build gensim dictionary\n",
    "        processed_texts = [text_tokenize(text) for text in texts]\n",
    "        dictionary = gensim.corpora.Dictionary(processed_texts)\n",
    "        \n",
    "        #build bag-of-words representation\n",
    "        bow = [dictionary.doc2bow(text) for text in processed_texts]\n",
    "        #tfidf = models.TfidfModel(bow)\n",
    "        #corpus_tfidf = tfidf[bow]\n",
    "    \n",
    "    \n",
    "        #guided lda with eta\n",
    "        if topic_word_priors and eta_ != None:\n",
    "            etas = []\n",
    "        \n",
    "            for r in range(len(topic_word_priors)):\n",
    "                eta = []\n",
    "                for i in range(len(dictionary)):\n",
    "                    \n",
    "                    if dictionary[i] in topic_word_priors[r]:\n",
    "                        eta.append(np.array(eta_))\n",
    "                    else:\n",
    "                        eta.append(np.array(1/topic_num))\n",
    "                etas.append(eta)\n",
    "    \n",
    "            model = gensim.models.ldamodel.LdaModel(\n",
    "                corpus=bow, id2word=dictionary, num_topics=topic_num,\n",
    "                random_state=42, chunksize=100, eta=np.array(etas),\n",
    "                eval_every=-1, update_every=1,\n",
    "                passes=150, alpha='auto', per_word_topics=True)\n",
    "    \n",
    "            #transform corpus into topics\n",
    "            transformed_corpus = model[bow]\n",
    "    \n",
    "            #extract topical terms\n",
    "            topical_terms = [[token for token in m[1].split('\"') if not re.findall(r'\\d',token)] for m in  model.show_topics(num_words=numwords)]\n",
    "            return model, topical_terms, transformed_corpus, processed_texts\n",
    "        \n",
    "        else:\n",
    "        #standard lda\n",
    "            model = gensim.models.ldamodel.LdaModel(\n",
    "                corpus=bow, id2word=dictionary, num_topics=topic_num,\n",
    "                random_state=42, chunksize=100, eta=None,\n",
    "                eval_every=-1, update_every=1,\n",
    "                passes=150, alpha='auto', per_word_topics=True)\n",
    "            \n",
    "            #transform corpus into topics\n",
    "            transformed_corpus = model[bow]\n",
    "            \n",
    "            #extract topical terms\n",
    "            topical_terms = [[token for token in m[1].split('\"') if not re.findall(r'\\d',token)] for m in  model.show_topics(num_words=25)]\n",
    "            return model, topical_terms, transformed_corpus, processed_texts\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aed20584-91e8-40b7-af9c-3a1307e1ca16",
   "metadata": {},
   "source": [
    "## Gensim Default Topic Modeling"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1fde9d99-a1c9-47a5-b884-162724f6be9b",
   "metadata": {},
   "source": [
    "### Gensim LDA Posts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "id": "6f8c5eea-b9be-4ef8-9e6a-0807c18a3e08",
   "metadata": {},
   "outputs": [],
   "source": [
    "gensim_post_model, gensim_post_terms, gensim_post_corpus, gensim_processed_posts = gensim_lda(posts, topic_num=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46694422-2d58-4b30-97bf-47580c309e71",
   "metadata": {},
   "source": [
    "### Gensim LDA Threads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "id": "22da19bd-422a-4708-b3c4-cedf188b1545",
   "metadata": {},
   "outputs": [],
   "source": [
    "gensim_thread_model, gensim_thread_terms, gensim_thread_corpus, gensim_processed_threads = gensim_lda(threads, topic_num=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "580b597b-a7eb-4404-abb2-9cfbbdaa429b",
   "metadata": {},
   "source": [
    "### Gensim Guided Post LDA High Degree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "id": "5cd53a13-2bb6-4766-8e5d-8bfd01fc9aa8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#gensim requires a list of tokens to set topic priors\n",
    "degree_priors = [text_tokenize(node) for node in [coronavirus_nodes, crime_nodes + metro_nodes, plan_nodes,retail_nodes]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "id": "cf279323-60fd-4ca3-84b0-d77f715924b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "gensim_degree_model, gensim_degree_terms, gensim_degree_corpus, gensim_degree_processed_posts = gensim_lda(posts, topic_num=4, topic_word_priors=degree_priors, eta_=1.0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2255828c-44de-4b9a-9264-384ccb7acf05",
   "metadata": {},
   "source": [
    "### Gensim Guided Threads LDA High Degree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "id": "a9a8a2dc-4b32-483f-b41b-45a180999793",
   "metadata": {},
   "outputs": [],
   "source": [
    "gensim_thread_degree_model, gensim__thread_degree_terms, gensim_thread_degree_corpus, gensim_thread_degree_processed_posts = gensim_lda(threads, topic_num=4, topic_word_priors=degree_priors, eta_=1.0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f2d78b2-c06e-41af-8915-a86cee5072b4",
   "metadata": {},
   "source": [
    "### Gensim Guided Posts LDA Initial Posts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "id": "fe67e96d-e7c7-46fc-a66d-346664cc7461",
   "metadata": {},
   "outputs": [],
   "source": [
    "first_posts_tokens = [text_tokenize(post) for post in first_posts]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "id": "a1e260f1-eefc-4bdc-a359-a4020548ad1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "gensim_init_post_model, gensim_init_posts_terms, gensim_init_posts_corpus, processed_posts_init_posts = gensim_lda(posts, topic_num=4, topic_word_priors=first_posts_tokens, eta_=1.0)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5ca6ca3-c335-4e2d-ae39-4e5f18597a81",
   "metadata": {},
   "source": [
    "### Gensim Guided Threads LDA Initial Posts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "id": "f833c5e1-6177-4b5f-b276-46ab1541e78c",
   "metadata": {},
   "outputs": [],
   "source": [
    "gensim_thread_init_post_model, gensim_thread_init_posts_terms, gensim__thread_init_posts_corpus, gensim_thread_processed_posts_init_posts = gensim_lda(threads, topic_num=4, topic_word_priors=first_posts_tokens, eta_=1.0)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a726a505-254d-4d98-bbcf-0710eaa54784",
   "metadata": {},
   "source": [
    "### Gensim Guided Post LDA Forum Titles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "id": "dbdc0834-a686-4e4a-b28a-cea8d70aee54",
   "metadata": {},
   "outputs": [],
   "source": [
    "title_terms = [text_tokenize(title) for title in titles.split('\\n')]\n",
    "\n",
    "gensim_titles_model, gensim_titles_terms, gensim_titles_corpus, gensim_titles_processed_posts = gensim_lda(posts, topic_num=4, topic_word_priors=title_terms, eta_=1.0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "159ccfc0-4bbd-445c-8f6a-f6014042cd7f",
   "metadata": {},
   "source": [
    "### Gensim Guided Threads LDA Forum Titles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "id": "f659d783-285d-4401-9222-8de7833cb410",
   "metadata": {},
   "outputs": [],
   "source": [
    "gensim_thread_titles_model, gensim_thread_titles_terms, gensim_thread_titles_corpus, gensim_thread_titles_processed_posts = gensim_lda(threads, topic_num=4, topic_word_priors=title_terms, eta_=1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "id": "cf7877e6-6405-4c71-8635-8045ddbcbcad",
   "metadata": {},
   "outputs": [],
   "source": [
    "gensim_eval_df = pd.concat([\n",
    "    get_coherence_diversity_scores(gensim_processed_posts, gensim_post_terms,fast,metric='u_mass',topic_type='lda posts'),\n",
    "    get_coherence_diversity_scores(gensim_titles_processed_posts, gensim_titles_terms,fast,metric='u_mass',topic_type='guided lda (post titles)'),\n",
    "    get_coherence_diversity_scores(gensim_degree_processed_posts, gensim_degree_terms, fast, metric='u_mass',topic_type='guided lda (high degree)'),\n",
    "    get_coherence_diversity_scores(processed_posts_init_posts, gensim_init_posts_terms,fast, metric=\"u_mass\",topic_type='guided lda (initial posts)'),\n",
    "    get_coherence_diversity_scores(gensim_processed_threads, gensim_thread_terms,fast,metric='u_mass',topic_type='lda threads'),\n",
    "    get_coherence_diversity_scores(gensim_thread_degree_processed_posts, gensim__thread_degree_terms,fast,metric='u_mass',topic_type='lda threads (high degree)'),\n",
    "    get_coherence_diversity_scores(gensim_thread_titles_processed_posts, gensim_thread_titles_terms,fast,metric='u_mass',topic_type='lda threads (post titles)'),\n",
    "    get_coherence_diversity_scores(gensim_thread_processed_posts_init_posts, gensim_thread_init_posts_terms,fast,metric='u_mass',topic_type='lda threads (initial post)'),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "id": "10635f47-1113-4065-8862-957218597b87",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>u_mass_coherence</th>\n",
       "      <th>puw:</th>\n",
       "      <th>jd:</th>\n",
       "      <th>we-cd:</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>lda posts</th>\n",
       "      <td>-2.049662</td>\n",
       "      <td>0.950</td>\n",
       "      <td>0.929901</td>\n",
       "      <td>0.234451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>guided lda (post titles)</th>\n",
       "      <td>-2.007422</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.956038</td>\n",
       "      <td>0.256340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>guided lda (high degree)</th>\n",
       "      <td>-2.175875</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.959436</td>\n",
       "      <td>0.262369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>guided lda (initial posts)</th>\n",
       "      <td>-2.126475</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.970196</td>\n",
       "      <td>0.273824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lda threads</th>\n",
       "      <td>-2.126444</td>\n",
       "      <td>0.900</td>\n",
       "      <td>0.944670</td>\n",
       "      <td>0.344369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lda threads (high degree)</th>\n",
       "      <td>-1.922054</td>\n",
       "      <td>0.875</td>\n",
       "      <td>0.910089</td>\n",
       "      <td>0.221785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lda threads (post titles)</th>\n",
       "      <td>-1.910564</td>\n",
       "      <td>0.875</td>\n",
       "      <td>0.937734</td>\n",
       "      <td>0.307599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lda threads (initial post)</th>\n",
       "      <td>-1.842810</td>\n",
       "      <td>0.925</td>\n",
       "      <td>0.930816</td>\n",
       "      <td>0.318339</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            u_mass_coherence   puw:       jd:    we-cd:\n",
       "lda posts                          -2.049662  0.950  0.929901  0.234451\n",
       "guided lda (post titles)           -2.007422  1.000  0.956038  0.256340\n",
       "guided lda (high degree)           -2.175875  1.000  0.959436  0.262369\n",
       "guided lda (initial posts)         -2.126475  0.975  0.970196  0.273824\n",
       "lda threads                        -2.126444  0.900  0.944670  0.344369\n",
       "lda threads (high degree)          -1.922054  0.875  0.910089  0.221785\n",
       "lda threads (post titles)          -1.910564  0.875  0.937734  0.307599\n",
       "lda threads (initial post)         -1.842810  0.925  0.930816  0.318339"
      ]
     },
     "execution_count": 222,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gensim_eval_df[['u_mass_coherence','puw:','jd:','we-cd:']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "id": "da0d1284-86c8-44ac-97d1-aed51bfd99ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>u_mass_coherence</th>\n",
       "      <th>puw:</th>\n",
       "      <th>jd:</th>\n",
       "      <th>we-cd:</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>-2.020163</td>\n",
       "      <td>0.937500</td>\n",
       "      <td>0.942360</td>\n",
       "      <td>0.277384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.120100</td>\n",
       "      <td>0.051755</td>\n",
       "      <td>0.019310</td>\n",
       "      <td>0.042582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-2.175875</td>\n",
       "      <td>0.875000</td>\n",
       "      <td>0.910089</td>\n",
       "      <td>0.221785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-2.126452</td>\n",
       "      <td>0.893750</td>\n",
       "      <td>0.930587</td>\n",
       "      <td>0.250868</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>-2.028542</td>\n",
       "      <td>0.937500</td>\n",
       "      <td>0.941202</td>\n",
       "      <td>0.268096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>-1.919181</td>\n",
       "      <td>0.981250</td>\n",
       "      <td>0.956887</td>\n",
       "      <td>0.310284</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>-1.842810</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.970196</td>\n",
       "      <td>0.344369</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       u_mass_coherence      puw:       jd:    we-cd:\n",
       "count          8.000000  8.000000  8.000000  8.000000\n",
       "mean          -2.020163  0.937500  0.942360  0.277384\n",
       "std            0.120100  0.051755  0.019310  0.042582\n",
       "min           -2.175875  0.875000  0.910089  0.221785\n",
       "25%           -2.126452  0.893750  0.930587  0.250868\n",
       "50%           -2.028542  0.937500  0.941202  0.268096\n",
       "75%           -1.919181  0.981250  0.956887  0.310284\n",
       "max           -1.842810  1.000000  0.970196  0.344369"
      ]
     },
     "execution_count": 223,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gensim_eval_df[['u_mass_coherence','puw:','jd:','we-cd:']].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "947766a9-b067-4e97-ae0b-df127575c6e5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}