Clustering-Metrics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Clustering Metrics\n",
    "\n",
    "Use saved-embeddings and labels for qna_5500 dataset to compute metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import os\n",
    "import numpy as np\n",
    "\n",
    "from clustering_utils import KMeans as CustomKMeans\n",
    "from sklearn.cluster import KMeans, AgglomerativeClustering\n",
    "from sklearn.mixture import GaussianMixture\n",
    "from sklearn.metrics import homogeneity_score, silhouette_score, adjusted_mutual_info_score\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_metrics(embed_arr, label_arr, num_clusters):\n",
    "    # Reference KMeans implementation\n",
    "    kmeans_ref = KMeans(n_clusters=num_clusters)\n",
    "    kmeans_ref.fit(embed_arr)\n",
    "    y_hat_ref = kmeans_ref.labels_\n",
    "\n",
    "    # Custom KMeans with cosine distance\n",
    "    kmeans_2 = CustomKMeans(num_clusters=num_clusters, dist_fn='cosine')\n",
    "    kmeans_2.fit(embed_arr)\n",
    "    y_hat_2 = kmeans_2.cluster_labels\n",
    "\n",
    "    # Agglomerative clustering: Default\n",
    "    agg_clust_1 = AgglomerativeClustering(n_clusters=num_clusters)\n",
    "    agg_clust_1.fit(embed_arr)\n",
    "    y_hat_3 = agg_clust_1.labels_\n",
    "\n",
    "    # GMM: Default\n",
    "    gmm_1 = GaussianMixture(n_components=num_clusters, covariance_type='full')\n",
    "    y_hat_4 = gmm_1.fit_predict(embed_arr)\n",
    "\n",
    "    # GMM: Diag\n",
    "    gmm_2 = GaussianMixture(n_components=num_clusters, covariance_type='diag')\n",
    "    y_hat_5 = gmm_2.fit_predict(embed_arr)\n",
    "\n",
    "    print('-' * 50)\n",
    "    print(f'Euclidean Reference H-score: {homogeneity_score(labels_true=label_arr, labels_pred=y_hat_ref)}')\n",
    "    print(f'Cosine KMeans H-score: {homogeneity_score(labels_true=label_arr, labels_pred=y_hat_2)}')\n",
    "    print(f'Euclidean Agg H-score: {homogeneity_score(labels_true=label_arr, labels_pred=y_hat_3)}')\n",
    "    print(f'GMM Full H-score: {homogeneity_score(labels_true=label_arr, labels_pred=y_hat_4)}')\n",
    "    print(f'GMM Diag H-score: {homogeneity_score(labels_true=label_arr, labels_pred=y_hat_5)}')\n",
    "    print('-' * 50)\n",
    "    print(f'Euclidean Reference AMI-score: {adjusted_mutual_info_score(labels_true=label_arr, labels_pred=y_hat_ref)}')\n",
    "    print(f'Cosine KMeans AMI-score: {adjusted_mutual_info_score(labels_true=label_arr, labels_pred=y_hat_2)}')\n",
    "    print(f'Euclidean Agg AMI-score: {adjusted_mutual_info_score(labels_true=label_arr, labels_pred=y_hat_3)}')\n",
    "    print(f'GMM Full AMI-score: {adjusted_mutual_info_score(labels_true=label_arr, labels_pred=y_hat_4)}')\n",
    "    print(f'GMM Diag AMI-score: {adjusted_mutual_info_score(labels_true=label_arr, labels_pred=y_hat_5)}')\n",
    "    print('-' * 50)\n",
    "    print(f'Euclidean Reference Silhouette-score: {silhouette_score(X=embed_arr, labels=y_hat_ref)}')\n",
    "    print(f'Cosine KMeans Silhouette-score: {silhouette_score(X=embed_arr, labels=y_hat_2)}')\n",
    "    print(f'Euclidean Agg Silhouette-score: {silhouette_score(X=embed_arr, labels=y_hat_3)}')\n",
    "    print(f'GMM Full Silhouette-score: {silhouette_score(X=embed_arr, labels=y_hat_4)}')\n",
    "    print(f'GMM Diag Silhouette-score: {silhouette_score(X=embed_arr, labels=y_hat_5)}')\n",
    "    print('-' * 50)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_CLUSTERS = 50\n",
    "\n",
    "label_arr = np.load(os.path.join('SavedEmbeddings', f'qna_5500_labels_{NUM_CLUSTERS}_classes.npy'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sentence Transformers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ST1 Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n",
      "Euclidean Reference H-score: 0.4201506964587305\n",
      "Cosine KMeans H-score: 0.4124560875757869\n",
      "Euclidean Agg H-score: 0.33232499744197597\n",
      "GMM Full H-score: 0.41519702438498046\n",
      "GMM Diag H-score: 0.4106238038561892\n",
      "--------------------------------------------------\n",
      "Euclidean Reference AMI-score: 0.3411417873261878\n",
      "Cosine KMeans AMI-score: 0.33271542759233197\n",
      "Euclidean Agg AMI-score: 0.26080777826959534\n",
      "GMM Full AMI-score: 0.3358286087229338\n",
      "GMM Diag AMI-score: 0.332303626078361\n",
      "--------------------------------------------------\n",
      "Euclidean Reference Silhouette-score: 0.03418366238474846\n",
      "Cosine KMeans Silhouette-score: 0.035480909049510956\n",
      "Euclidean Agg Silhouette-score: 0.014664233662188053\n",
      "GMM Full Silhouette-score: 0.033146899193525314\n",
      "GMM Diag Silhouette-score: 0.028425663709640503\n",
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "embed_arr = np.load(os.path.join('SavedEmbeddings', 'qna_5500_embeddings_ST1.npy'))\n",
    "\n",
    "print_metrics(embed_arr, label_arr, NUM_CLUSTERS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ST2 Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n",
      "Euclidean Reference H-score: 0.3869684978451748\n",
      "Cosine KMeans H-score: 0.39232659758759564\n",
      "Euclidean Agg H-score: 0.32838580337542767\n",
      "GMM Full H-score: 0.3760716629250223\n",
      "GMM Diag H-score: 0.39897155309474236\n",
      "--------------------------------------------------\n",
      "Euclidean Reference AMI-score: 0.30911164061849833\n",
      "Cosine KMeans AMI-score: 0.3138350903847671\n",
      "Euclidean Agg AMI-score: 0.25706726065782104\n",
      "GMM Full AMI-score: 0.3004686457167296\n",
      "GMM Diag AMI-score: 0.3214921829371091\n",
      "--------------------------------------------------\n",
      "Euclidean Reference Silhouette-score: 0.023418495431542397\n",
      "Cosine KMeans Silhouette-score: 0.025374358519911766\n",
      "Euclidean Agg Silhouette-score: 0.008664585649967194\n",
      "GMM Full Silhouette-score: 0.02190873771905899\n",
      "GMM Diag Silhouette-score: 0.011084144003689289\n",
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "embed_arr = np.load(os.path.join('SavedEmbeddings', 'qna_5500_embeddings_ST2.npy'))\n",
    "\n",
    "print_metrics(embed_arr, label_arr, NUM_CLUSTERS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ST3 Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n",
      "Euclidean Reference H-score: 0.41944889088258086\n",
      "Cosine KMeans H-score: 0.41671297401005497\n",
      "Euclidean Agg H-score: 0.33175594122154023\n",
      "GMM Full H-score: 0.4237260981922054\n",
      "GMM Diag H-score: 0.4384880367138227\n",
      "--------------------------------------------------\n",
      "Euclidean Reference AMI-score: 0.3391017894781413\n",
      "Cosine KMeans AMI-score: 0.33604840032835914\n",
      "Euclidean Agg AMI-score: 0.2610903438737921\n",
      "GMM Full AMI-score: 0.3451370559250015\n",
      "GMM Diag AMI-score: 0.36003742786232934\n",
      "--------------------------------------------------\n",
      "Euclidean Reference Silhouette-score: 0.025794342160224915\n",
      "Cosine KMeans Silhouette-score: 0.024783072993159294\n",
      "Euclidean Agg Silhouette-score: 0.005842156708240509\n",
      "GMM Full Silhouette-score: 0.024312039837241173\n",
      "GMM Diag Silhouette-score: 0.02287040650844574\n",
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "embed_arr = np.load(os.path.join('SavedEmbeddings', 'qna_5500_embeddings_ST3.npy'))\n",
    "\n",
    "print_metrics(embed_arr, label_arr, NUM_CLUSTERS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Universal Sentence Encoder Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n",
      "Euclidean Reference H-score: 0.5040072749493797\n",
      "Cosine KMeans H-score: 0.4955793407121537\n",
      "Euclidean Agg H-score: 0.36225533691976036\n",
      "GMM Full H-score: 0.5005220323748278\n",
      "GMM Diag H-score: 0.5231154644634167\n",
      "--------------------------------------------------\n",
      "Euclidean Reference AMI-score: 0.42335196770218714\n",
      "Cosine KMeans AMI-score: 0.41352281799803003\n",
      "Euclidean Agg AMI-score: 0.2905312839484957\n",
      "GMM Full AMI-score: 0.41884699438294093\n",
      "GMM Diag AMI-score: 0.4425918153491631\n",
      "--------------------------------------------------\n",
      "Euclidean Reference Silhouette-score: 0.02817135490477085\n",
      "Cosine KMeans Silhouette-score: 0.031381502747535706\n",
      "Euclidean Agg Silhouette-score: 0.0026244723703712225\n",
      "GMM Full Silhouette-score: 0.026338528841733932\n",
      "GMM Diag Silhouette-score: 0.023362521082162857\n",
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "embed_arr = np.load(os.path.join('SavedEmbeddings', 'qna_5500_embeddings_USE.npy'))\n",
    "\n",
    "print_metrics(embed_arr, label_arr, NUM_CLUSTERS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## InferSent Metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### IS1 Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n",
      "Euclidean Reference H-score: 0.3169752744416322\n",
      "Cosine KMeans H-score: 0.3380504537725456\n",
      "Euclidean Agg H-score: 0.3166454529590655\n",
      "GMM Full H-score: 0.33028774347829226\n",
      "GMM Diag H-score: 0.3367380979734525\n",
      "--------------------------------------------------\n",
      "Euclidean Reference AMI-score: 0.24114859444539127\n",
      "Cosine KMeans AMI-score: 0.26143929757499823\n",
      "Euclidean Agg AMI-score: 0.24309886415628504\n",
      "GMM Full AMI-score: 0.2542032811610792\n",
      "GMM Diag AMI-score: 0.2601474999541236\n",
      "--------------------------------------------------\n",
      "Euclidean Reference Silhouette-score: 0.022250670939683914\n",
      "Cosine KMeans Silhouette-score: 0.016235917806625366\n",
      "Euclidean Agg Silhouette-score: 0.00029120355611667037\n",
      "GMM Full Silhouette-score: 0.023101061582565308\n",
      "GMM Diag Silhouette-score: 0.01788642629981041\n",
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "embed_arr = np.load(os.path.join('SavedEmbeddings', 'qna_5500_embeddings_IS1.npy'))\n",
    "\n",
    "print_metrics(embed_arr, label_arr, NUM_CLUSTERS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### IS2 Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n",
      "Euclidean Reference H-score: 0.34530911362472577\n",
      "Cosine KMeans H-score: 0.3431478823662465\n",
      "Euclidean Agg H-score: 0.3270207381106829\n",
      "GMM Full H-score: 0.3338484584969446\n",
      "GMM Diag H-score: 0.3701990563270837\n",
      "--------------------------------------------------\n",
      "Euclidean Reference AMI-score: 0.2679084731406709\n",
      "Cosine KMeans AMI-score: 0.26746715400308446\n",
      "Euclidean Agg AMI-score: 0.2560798772042494\n",
      "GMM Full AMI-score: 0.25892687701549894\n",
      "GMM Diag AMI-score: 0.2945399588688473\n",
      "--------------------------------------------------\n",
      "Euclidean Reference Silhouette-score: 0.0208977572619915\n",
      "Cosine KMeans Silhouette-score: 0.02219308353960514\n",
      "Euclidean Agg Silhouette-score: 0.0011304031359031796\n",
      "GMM Full Silhouette-score: 0.020395392552018166\n",
      "GMM Diag Silhouette-score: 0.014055631123483181\n",
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "embed_arr = np.load(os.path.join('SavedEmbeddings', 'qna_5500_embeddings_IS2.npy'))\n",
    "\n",
    "print_metrics(embed_arr, label_arr, NUM_CLUSTERS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fused embedding metrics\n",
    "\n",
    "### ST1 | ST3 | USE : 70% train set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n",
      "Euclidean Reference H-score: 0.880788874886895\n",
      "Cosine KMeans H-score: 0.8697534576925424\n",
      "Euclidean Agg H-score: 0.8666451908145861\n",
      "GMM Full H-score: 0.8668550966400141\n",
      "GMM Diag H-score: 0.8491107575085236\n",
      "--------------------------------------------------\n",
      "Euclidean Reference AMI-score: 0.790192349143632\n",
      "Cosine KMeans AMI-score: 0.7812895299924696\n",
      "Euclidean Agg AMI-score: 0.7926231495380615\n",
      "GMM Full AMI-score: 0.7744596814949175\n",
      "GMM Diag AMI-score: 0.7608170878900994\n",
      "--------------------------------------------------\n",
      "Euclidean Reference Silhouette-score: 0.0824858546257019\n",
      "Cosine KMeans Silhouette-score: 0.07075781375169754\n",
      "Euclidean Agg Silhouette-score: 0.07374074310064316\n",
      "GMM Full Silhouette-score: 0.07567330449819565\n",
      "GMM Diag Silhouette-score: 0.06903542578220367\n",
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "embed_arr = np.load(os.path.join('SavedEmbeddings', 'qna_5500_embeddings_ST1_ST3_USE_old.npy'))\n",
    "\n",
    "print_metrics(embed_arr, label_arr, NUM_CLUSTERS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ST1 | USE: 70% train set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n",
      "Euclidean Reference H-score: 0.8530434152267252\n",
      "Cosine KMeans H-score: 0.8611816414070377\n",
      "Euclidean Agg H-score: 0.8572878531032125\n",
      "GMM Full H-score: 0.8512132962253891\n",
      "GMM Diag H-score: 0.8581067851326362\n",
      "--------------------------------------------------\n",
      "Euclidean Reference AMI-score: 0.7576955123743465\n",
      "Cosine KMeans AMI-score: 0.7686270860734354\n",
      "Euclidean Agg AMI-score: 0.7793438309992649\n",
      "GMM Full AMI-score: 0.7631381818686135\n",
      "GMM Diag AMI-score: 0.7654717570364103\n",
      "--------------------------------------------------\n",
      "Euclidean Reference Silhouette-score: 0.06750435382127762\n",
      "Cosine KMeans Silhouette-score: 0.0725121721625328\n",
      "Euclidean Agg Silhouette-score: 0.07072628289461136\n",
      "GMM Full Silhouette-score: 0.06964422762393951\n",
      "GMM Diag Silhouette-score: 0.07476882636547089\n",
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "embed_arr = np.load(os.path.join('SavedEmbeddings', 'qna_5500_embeddings_ST1_USE.npy'))\n",
    "\n",
    "print_metrics(embed_arr, label_arr, NUM_CLUSTERS)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6.13 ('6220_env')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "cf5c57b28b59ef54413eec753c1727813b581157cd2569c79e355e33e8c0aafc"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}