generic.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import metrics\n",
    "\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.cluster import AgglomerativeClustering\n",
    "from sklearn.cluster import DBSCAN\n",
    "from sklearn.cluster import MeanShift\n",
    "from sklearn.cluster import Birch\n",
    "from sklearn.cluster import AffinityPropagation\n",
    "from sklearn.cluster import MiniBatchKMeans\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/generic.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>text</th>\n",
       "      <th>class</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>444</td>\n",
       "      <td>In need of a treat? We've got just the thing. ...</td>\n",
       "      <td>win</td>\n",
       "      <td>1324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>305</td>\n",
       "      <td>Acme Works is closed this week so we can ring ...</td>\n",
       "      <td>startup</td>\n",
       "      <td>1185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>146</td>\n",
       "      <td>Be strong enough to let go, and patient enough...</td>\n",
       "      <td>quote</td>\n",
       "      <td>1026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>354</td>\n",
       "      <td>Congratulations to BrainCeek, one of this year...</td>\n",
       "      <td>startup</td>\n",
       "      <td>1234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>466</td>\n",
       "      <td>I love The Office! #WIN</td>\n",
       "      <td>win</td>\n",
       "      <td>1346</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                               text    class  \\\n",
       "0         444  In need of a treat? We've got just the thing. ...      win   \n",
       "1         305  Acme Works is closed this week so we can ring ...  startup   \n",
       "2         146  Be strong enough to let go, and patient enough...    quote   \n",
       "3         354  Congratulations to BrainCeek, one of this year...  startup   \n",
       "4         466                            I love The Office! #WIN      win   \n",
       "\n",
       "     id  \n",
       "0  1324  \n",
       "1  1185  \n",
       "2  1026  \n",
       "3  1234  \n",
       "4  1346  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.sample(frac=1).reset_index(drop=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(500, 4)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>text</th>\n",
       "      <th>class</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>444</td>\n",
       "      <td>In need of a treat? We've got just the thing. ...</td>\n",
       "      <td>4</td>\n",
       "      <td>1324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>305</td>\n",
       "      <td>Acme Works is closed this week so we can ring ...</td>\n",
       "      <td>2</td>\n",
       "      <td>1185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>146</td>\n",
       "      <td>Be strong enough to let go, and patient enough...</td>\n",
       "      <td>1</td>\n",
       "      <td>1026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>354</td>\n",
       "      <td>Congratulations to BrainCeek, one of this year...</td>\n",
       "      <td>2</td>\n",
       "      <td>1234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>466</td>\n",
       "      <td>I love The Office! #WIN</td>\n",
       "      <td>4</td>\n",
       "      <td>1346</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                               text  class    id\n",
       "0         444  In need of a treat? We've got just the thing. ...      4  1324\n",
       "1         305  Acme Works is closed this week so we can ring ...      2  1185\n",
       "2         146  Be strong enough to let go, and patient enough...      1  1026\n",
       "3         354  Congratulations to BrainCeek, one of this year...      2  1234\n",
       "4         466                            I love The Office! #WIN      4  1346"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import preprocessing\n",
    "\n",
    "label_encoding = preprocessing.LabelEncoder()\n",
    "\n",
    "df['class'] = label_encoding.fit_transform(df['class'].astype(str))\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38     1\n",
       "99     4\n",
       "472    3\n",
       "262    2\n",
       "212    3\n",
       "264    0\n",
       "46     0\n",
       "19     0\n",
       "476    4\n",
       "336    2\n",
       "Name: class, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_labels = df['class']\n",
    "\n",
    "df_labels.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(clustering_model, data, labels):\n",
    "    \n",
    "    model = clustering_model(data)\n",
    "\n",
    "    print('homo\\tcompl\\tv-meas\\tARI\\tAMI\\tsilhouette')\n",
    "    print(50 * '-')\n",
    "    \n",
    "    print('%.3f\\t%.3f\\t%.3f\\t%.3f\\t%.3f\\t%.3f'\n",
    "          %(metrics.homogeneity_score(labels, model.labels_),\n",
    "            metrics.completeness_score(labels, model.labels_),\n",
    "            metrics.v_measure_score(labels, model.labels_),\n",
    "            metrics.adjusted_rand_score(labels, model.labels_),\n",
    "            metrics.adjusted_mutual_info_score(labels,  model.labels_),\n",
    "            metrics.silhouette_score(data, model.labels_)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def k_means(data, n_clusters=5, max_iter=1000):\n",
    "    model = KMeans(n_clusters=n_clusters, max_iter=max_iter).fit(data)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def agglomerative_fn(data, n_clusters=5):\n",
    "    model = AgglomerativeClustering(n_clusters = n_clusters).fit(data)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dbscan_fn(data, eps=0.45, min_samples=4):\n",
    "    model = DBSCAN(eps=eps, min_samples=min_samples).fit(data)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mean_shift_fn(data, bandwidth=0.85):\n",
    "    model = MeanShift(bandwidth=bandwidth).fit(data)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def birch_fn(data, n_clusters=5):\n",
    "    model = Birch(n_clusters=n_clusters).fit(data)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def affinity_propagation_fn(data, damping=0.6, max_iter=1000):\n",
    "    model = AffinityPropagation(damping=damping, max_iter=max_iter).fit(data)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mini_batch_kmeans_fn(data, n_clusters=5, max_iter=1000):\n",
    "    model = MiniBatchKMeans(n_clusters=n_clusters, max_iter=max_iter, batch_size=20).fit(data)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TFIDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "tfidf_vectorizer=TfidfVectorizer(stop_words='english')\n",
    "df_tfidf=tfidf_vectorizer.fit_transform(df['text']) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<500x3130 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 6112 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "homo\tcompl\tv-meas\tARI\tAMI\tsilhouette\n",
      "--------------------------------------------------\n",
      "0.838\t0.840\t0.839\t0.848\t0.837\t0.015\n"
     ]
    }
   ],
   "source": [
    "build_model(k_means, df_tfidf.toarray(), df_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "homo\tcompl\tv-meas\tARI\tAMI\tsilhouette\n",
      "--------------------------------------------------\n",
      "0.176\t0.324\t0.228\t0.086\t0.217\t0.010\n"
     ]
    }
   ],
   "source": [
    "build_model(agglomerative_fn, df_tfidf.toarray(), df_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "homo\tcompl\tv-meas\tARI\tAMI\tsilhouette\n",
      "--------------------------------------------------\n",
      "0.016\t0.282\t0.031\t0.001\t0.020\t0.008\n"
     ]
    }
   ],
   "source": [
    "build_model(dbscan_fn, df_tfidf.toarray(), df_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "homo\tcompl\tv-meas\tARI\tAMI\tsilhouette\n",
      "--------------------------------------------------\n",
      "0.998\t0.260\t0.413\t0.001\t0.012\t0.021\n"
     ]
    }
   ],
   "source": [
    "build_model(mean_shift_fn, df_tfidf.toarray(), df_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "homo\tcompl\tv-meas\tARI\tAMI\tsilhouette\n",
      "--------------------------------------------------\n",
      "0.300\t0.407\t0.345\t0.165\t0.337\t0.005\n"
     ]
    }
   ],
   "source": [
    "build_model(birch_fn, df_tfidf.toarray(), df_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "homo\tcompl\tv-meas\tARI\tAMI\tsilhouette\n",
      "--------------------------------------------------\n",
      "0.685\t0.283\t0.401\t0.163\t0.321\t0.040\n"
     ]
    }
   ],
   "source": [
    "build_model(affinity_propagation_fn, df_tfidf.toarray(), df_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "homo\tcompl\tv-meas\tARI\tAMI\tsilhouette\n",
      "--------------------------------------------------\n",
      "0.719\t0.764\t0.741\t0.679\t0.739\t0.012\n"
     ]
    }
   ],
   "source": [
    "build_model(mini_batch_kmeans_fn, df_tfidf.toarray(), df_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Doc2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from d2v_pre import d2v_pre\n",
    "from getEmbeddings import getEmbeddings\n",
    "import numpy as np\n",
    "\n",
    "d2v_pre('data/generic.csv')\n",
    "\n",
    "x,y = getEmbeddings('data/generic_doc2vec.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def doc2vec_build_model(clustering_model, data, labels):\n",
    "    \n",
    "    model = clustering_model(data)\n",
    "\n",
    "    print('homo\\tcompl\\tv-meas\\tARI\\tAMI')\n",
    "    print(50 * '-')\n",
    "    \n",
    "    print('%.3f\\t%.3f\\t%.3f\\t%.3f\\t%.3f'\n",
    "          %(metrics.homogeneity_score(labels, model.labels_),\n",
    "            metrics.completeness_score(labels, model.labels_),\n",
    "            metrics.v_measure_score(labels, model.labels_),\n",
    "            metrics.adjusted_rand_score(labels, model.labels_),\n",
    "            metrics.adjusted_mutual_info_score(labels,  model.labels_)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "homo\tcompl\tv-meas\tARI\tAMI\n",
      "--------------------------------------------------\n",
      "0.011\t0.012\t0.011\t0.001\t0.001\n"
     ]
    }
   ],
   "source": [
    "doc2vec_build_model(k_means, x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "homo\tcompl\tv-meas\tARI\tAMI\n",
      "--------------------------------------------------\n",
      "0.007\t0.007\t0.007\t-0.002\t-0.003\n"
     ]
    }
   ],
   "source": [
    "doc2vec_build_model(agglomerative_fn, x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "homo\tcompl\tv-meas\tARI\tAMI\n",
      "--------------------------------------------------\n",
      "0.000\t1.000\t0.000\t0.000\t0.000\n"
     ]
    }
   ],
   "source": [
    "doc2vec_build_model(dbscan_fn, x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "homo\tcompl\tv-meas\tARI\tAMI\n",
      "--------------------------------------------------\n",
      "0.000\t1.000\t0.000\t0.000\t0.000\n"
     ]
    }
   ],
   "source": [
    "doc2vec_build_model(mean_shift_fn, x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "homo\tcompl\tv-meas\tARI\tAMI\n",
      "--------------------------------------------------\n",
      "0.000\t1.000\t0.000\t0.000\t0.000\n"
     ]
    }
   ],
   "source": [
    "doc2vec_build_model(birch_fn,x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "homo\tcompl\tv-meas\tARI\tAMI\n",
      "--------------------------------------------------\n",
      "0.107\t0.052\t0.070\t0.002\t0.013\n"
     ]
    }
   ],
   "source": [
    "doc2vec_build_model(affinity_propagation_fn, x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "homo\tcompl\tv-meas\tARI\tAMI\n",
      "--------------------------------------------------\n",
      "0.008\t0.015\t0.011\t-0.002\t-0.004\n"
     ]
    }
   ],
   "source": [
    "doc2vec_build_model(mini_batch_kmeans_fn, x, y)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}