Comparison_to_PCA.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Comparison of PCA+DBSCAN to UMAP\n",
    "\n",
    "This can be [run in Google Colab](https://colab.research.google.com/github/jreades/ph-word-embeddings/blob/main/Comparison_to_PCA.ipynb)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle\n",
    "import math\n",
    "import re\n",
    "import os\n",
    "\n",
    "from wordcloud import WordCloud\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction import text\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.cluster import DBSCAN\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import cm\n",
    "from matplotlib.colors import ListedColormap, LinearSegmentedColormap\n",
    "\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "try:\n",
    "    import en_core_web_lg\n",
    "    nlp = en_core_web_lg.load()\n",
    "except (ModuleNotFoundError, OSError) as e:\n",
    "    import spacy.cli\n",
    "    spacy.cli.download(\"en_core_web_lg\")\n",
    "    import en_core_web_lg\n",
    "    nlp = en_core_web_lg.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    import umap\n",
    "except ModuleNotFoundError:\n",
    "    !pip install umap-learn\n",
    "    import umap"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Process Tutorial Data \n",
    "\n",
    "For demonstration purposes, here we pick up with Part 2 of the ['Clustering with sklearn'](https://programminghistorian.org/en/lessons/clustering-with-scikit-learn-in-python#1-loading-the-dataset--exploratory-data-analysis) tutorial on The Programming Historian."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ddf = pd.read_csv('https://raw.githubusercontent.com/programminghistorian/jekyll/gh-pages/assets/clustering-with-scikit-learn-in-python/data/RELIGION_abstracts.csv', \n",
    "                 usecols=['title','abstract','link','volume'])\n",
    "ddf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ddf.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lemmatizeAbstracts(x):\n",
    "        doc = nlp(x)\n",
    "        new_text = []\n",
    "        for token in doc:\n",
    "            new_text.append(token.lemma_)\n",
    "        text_string = \" \".join(new_text)\n",
    "        # getting rid of non-word characters\n",
    "        text_string = re.sub(r\"[^\\w\\s]+\", \"\", text_string)\n",
    "        text_string = re.sub(r\"\\s{2,}\", \" \", text_string)\n",
    "        return text_string\n",
    "\n",
    "ddf[\"abstract_lemma\"] = ddf.abstract.apply(lemmatizeAbstracts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf = TfidfVectorizer(stop_words=\"english\", ngram_range=(1,2), max_features=250, strip_accents=\"unicode\", min_df=10, max_df=200)\n",
    "tfidf_religion_array = tfidf.fit_transform(ddf[\"abstract_lemma\"])\n",
    "df_abstracts_tfidf = pd.DataFrame(tfidf_religion_array.toarray(), index=ddf.index, columns=tfidf.get_feature_names_out())\n",
    "df_abstracts_tfidf.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# using PCA to reduce the dimensionality\n",
    "pca = PCA(n_components=10, whiten=False, random_state=42)\n",
    "abstracts_pca = pca.fit_transform(df_abstracts_tfidf)\n",
    "df_abstracts_pca = pd.DataFrame(data=abstracts_pca)\n",
    "df_abstracts_pca.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here's the issue with PCA:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f\"Total explained variance is {np.sum(pca.explained_variance_)*100:0.2f}% with first eigenvector explaining {pca.explained_variance_[0]*100:0.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# I cannot reproduce the 4-cluster result using an eps of 0.2\n",
    "dbscan = DBSCAN(eps=0.2, metric=\"euclidean\")\n",
    "dbscan_labels = dbscan.fit_predict(df_abstracts_pca)\n",
    "df_abstracts_dbscan = ddf.copy()\n",
    "df_abstracts_dbscan['cluster'] = dbscan_labels\n",
    "df_abstracts_dbscan.cluster.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### UMAP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reducer = umap.UMAP(\n",
    "            n_neighbors=8,\n",
    "            min_dist=0.02,\n",
    "            n_components=2,\n",
    "            random_state=42)\n",
    "    \n",
    "# Basically reduces our 300 feature vectors for each thesis, down to n dimensions\n",
    "X_embedded = reducer.fit_transform(df_abstracts_tfidf)\n",
    "print(f\"Resulting embedding is: {X_embedded.shape[0]} rows by {X_embedded.shape[1]} columns.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Grab the DBSCAN assignments based on the \n",
    "# PCA-decomposed data\n",
    "f,axs = plt.subplots(1,1,figsize=(12,6))\n",
    "f.suptitle(\"UMAP Output\") \n",
    "\n",
    "if isinstance(axs, np.ndarray):\n",
    "    axs = axs.reshape(-1)\n",
    "else:\n",
    "    axs = [axs]\n",
    "\n",
    "newcolors = np.insert(cm.get_cmap('tab10', 5).colors, 0, [0.6, 0.6, 0.6, 0.7], axis=0)\n",
    "newcmp = ListedColormap(newcolors)\n",
    "    \n",
    "for i, ax in enumerate(axs):\n",
    "    ax.set_xlabel(f\"Dimension {i*2+1}\")\n",
    "    ax.set_ylabel(f\"Dimension {i*2+2}\")\n",
    "    sctr = ax.scatter(x=X_embedded[:,i*2], y=X_embedded[:,i*2+1], s=8, c=dbscan_labels, cmap=newcmp)\n",
    "    ax.legend(*sctr.legend_elements(), loc='upper left', title='PCA-Derived\\nDBSCAN Clusters')\n",
    "    \n",
    "f.tight_layout()\n",
    "#plt.savefig(os.path.join('UMAP_Output.png'), dpi=150)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fg_list = ['darkgrey','darkorange','lightblue','brown','red','green']\n",
    "fp = os.path.join(os.sep,'Library','Font','Khula-Light.ttf')\n",
    "bg = 'white'\n",
    "wd = 50\n",
    "\n",
    "def get_cloud(fg='black'):\n",
    "    return WordCloud(\n",
    "        #font_path=fp,\n",
    "        max_words=wd,\n",
    "        width=1000, height=1000,\n",
    "        mode='RGBA',\n",
    "        background_color=bg,\n",
    "        color_func=lambda *args ,**kwargs: fg,\n",
    "        stopwords=['religion','religious','article','study','paper'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# From https://stackoverflow.com/a/45096142\n",
    "stopwords=['religion','religious','article','study','paper','new','use','research','analysis']\n",
    "stpw = text.ENGLISH_STOP_WORDS.union(stopwords)\n",
    "\n",
    "vectorizer = TfidfVectorizer(stop_words=stpw)\n",
    "\n",
    "num_clouds = df_abstracts_dbscan.cluster.max()+1 # Counts from zero\n",
    "\n",
    "f,axs = plt.subplots(math.ceil(num_clouds/2),2,figsize=(12,12))\n",
    "\n",
    "for i in range(0,max(dbscan_labels)+1):\n",
    "    cldf = df_abstracts_dbscan[df_abstracts_dbscan.cluster==i]\n",
    "    vecs = vectorizer.fit_transform(cldf.abstract_lemma)\n",
    "    feature_names = vectorizer.get_feature_names_out()\n",
    "    dense = vecs.todense()\n",
    "    lst1 = dense.tolist()\n",
    "    df = pd.DataFrame(lst1, columns=feature_names)\n",
    "    wordcloud = get_cloud().generate_from_frequencies(df.T.sum(axis=1))\n",
    "    axs.reshape(-1)[i].imshow(wordcloud, interpolation='bilinear')\n",
    "    axs.reshape(-1)[i].axis('off')\n",
    "    axs.reshape(-1)[i].set_title(f'Cluster {i}')\n",
    "\n",
    "plt.axis('off')\n",
    "plt.suptitle(\"Cluster TF/IDF\")\n",
    "plt.tight_layout(pad=1.35)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth',150)\n",
    "for i in range(0,4):\n",
    "    print(f\"Cluster {i}\")\n",
    "    egs = dfl[dfl.cluster==i].title.sample(5, random_state=42)\n",
    "    for e in egs:\n",
    "        print(f\"\\tExample: {e[:75]}\")\n",
    "    print()\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}