K-Means_Agglomerative_and_DbScan.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn as sk\n",
    "from sklearn.cluster import AgglomerativeClustering\n",
    "from sklearn.cluster import KMeans\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "from sklearn import metrics\n",
    "\n",
    "from sklearn.cluster import DBSCAN\n",
    "from sklearn.preprocessing import MinMaxScaler\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_clusts = range(40)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = pd.read_csv(os.getcwd() + '/top5_chars.csv')\n",
    "X.index = X.iloc[:,0]\n",
    "X = X.dropna()\n",
    "\n",
    "X.drop('Unnamed: 0',axis=1,inplace=True)\n",
    "X.drop(['cursewords','FK','0','tot_sents'],axis=1,inplace=True)\n",
    "\n",
    "scaler = MinMaxScaler()\n",
    "X = scaler.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#so, we will actually have to break these guys down into senteces... \n",
    "# so I will have to do the sentence tokenizer :) \n",
    "def jacardian_distance(document_1_data, document_2_data):\n",
    "\n",
    "    \n",
    "    #for key in processed_article_hash.keys(): \n",
    "    #art_uniq_ws[key] = []\n",
    "    #for sent in processed_article_hash[key]:\n",
    "    #    art_uniq_ws[key].extend(sent)\n",
    "    #    art_uniq_ws[key] = list(set(art_uniq_ws[key]))\n",
    "    d1_uniqs = []\n",
    "    for sent in document_1_data:\n",
    "        d1_uniqs.extend(sent)\n",
    "        d1_uniqs = list(set(d1_uniqs))\n",
    "    d2_uniqs = []\n",
    "    for sent in document_2_data:\n",
    "        d2_uniqs.extend(sent)\n",
    "        d2_uniqs = list(set(d2_uniqs))\n",
    "        \n",
    "    words_in_doc_1_not_in_doc_2 = [value for value in d1_uniqs if value not in d2_uniqs]\n",
    "    words_in_doc_2_not_in_doc_1 = [value for value in d2_uniqs if value not in d1_uniqs]\n",
    "    words_in_both_doc_1_and_doc_2 = [value for value in d1_uniqs if value in d2_uniqs]\n",
    "    \n",
    "    \n",
    "    \n",
    "    #overlap = [value for value in d1_uniqs if value in d2_uniqs] \n",
    "    #in1Not2 = [value for value in d1_uniqs if value not in d2_uniqs]\n",
    "    #in2Not1 = [value for value in d2_uniqs if value not in d1_uniqs]\n",
    "   # print(len(in1Not2))\n",
    "    #print(len(in2Not1))\n",
    "    #print(len(sv))\n",
    "   # print(len(sv)+len(in2Not1)+len(in1Not2))\n",
    "\n",
    "   # sv = d2_uniqs + d1_uniqs\n",
    "   # sv = list(set(sv))\n",
    "   # print(len(overlap))\n",
    "    \n",
    "    \n",
    "    \n",
    "    jacardian = len(words_in_both_doc_1_and_doc_2)/(len(words_in_both_doc_1_and_doc_2) + len(words_in_doc_1_not_in_doc_2) + len(words_in_doc_2_not_in_doc_1)) # divide the counts appropiately\n",
    "    \n",
    "    return jacardian\n",
    "    \n",
    "    \n",
    "    \n",
    "#Cosine Similarity: Here we want to create vector representations for each document. Specifically, we want to come up with a vector that is based on the list of all words that occur across both documents. Then for each document we will create a vector that includes the counts of the number of time a word occurs in the document.\n",
    "\n",
    "#So if the document 1 is: \"the ship sails at midnight\" and document 2 is: \"the crow flies at noon.\" We would be creating a vector like: [the, ship, sails, at, midnight, crow, flies, noon]. Then we would calculate the values of the vector for each document. For document 1: [1,1,1,1,1,0,0,0] and for document 2: [1,0,0,1,0,1,1,1]. With these two vectors we would simply take the dot product and that would provide the cosine similarity. \n",
    "\n",
    "def cosine_similarity(document_1_data, document_2_data):\n",
    "    document_vector_word_index = [] # here fill this with an ordered list of all the unique words across both documents\n",
    "    d1_uniqs = []\n",
    "    d1_nun = []\n",
    "    for sent in document_1_data:\n",
    "        d1_uniqs.extend(sent)\n",
    "        d1_nun.extend(sent)\n",
    "        d1_uniqs = list(set(d1_uniqs))\n",
    "    d2_uniqs = []\n",
    "    d2_nun = []\n",
    "    for sent in document_2_data:\n",
    "        d2_uniqs.extend(sent)\n",
    "        d2_uniqs = list(set(d2_uniqs))\n",
    "        d2_nun.extend(sent)\n",
    "\n",
    "    \n",
    "    all_words = d1_nun+d2_nun   \n",
    "\n",
    "    overlap = d1_uniqs + d2_uniqs \n",
    "    # so this is all of the unique words across the two documents\n",
    "    document_vector_word_index += list(set(overlap))\n",
    "    \n",
    "\n",
    "    document_1_vector = list(np.zeros(len(document_vector_word_index)))  # fill in the array with the frequency of the words in the document\n",
    "    document_2_vector = list(np.zeros(len(document_vector_word_index))) # fill in the array with the frequency of the words in the document\n",
    "\n",
    "    for wOrd in d1_nun:\n",
    "        document_1_vector[document_vector_word_index.index(wOrd)] +=1\n",
    "    \n",
    "    for wOrd in d2_nun:\n",
    "        document_2_vector[document_vector_word_index.index(wOrd)] +=1\n",
    "    \n",
    "    d1_norm = (np.sum(np.array(document_1_vector)**2))**(1/2)\n",
    "    d2_norm = (np.sum(np.array(document_2_vector)**2))**(1/2)\n",
    "    \n",
    "    cosSim = np.dot(document_1_vector,document_2_vector)/(d1_norm*d2_norm)\n",
    "\n",
    "\n",
    "    \n",
    "    return (cosSim)\n",
    "#document_1_vector,document_2_vector,document_vector_word_index, are the things I tried to return in testing\n",
    "#dot_product_of_two_document_vectors # you can refer to the numpy information on how to calculate the dot product of vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a variable to store your table data... you could use a hash or some other data structure. \n",
    "# We just want it to identify which document is being compared to which other document.\n",
    "\n",
    "data_structure_for_jacard_similarity = []\n",
    "data_structure_for_cosine_similarity = []\n",
    "# so I think what I'm ACTUALLY going to do here \n",
    "# is just append a new column\n",
    "# and then say the column is .... \n",
    "# so we are comparing each character to every OTHER character (including themselves)\n",
    "for char_1_index in char_series.index:\n",
    "    for char_2_index in char_series.index:\n",
    "        # we have the nested for loops as one way to compare each document to each other document\n",
    "        data_structure_for_jacard_similarity.append(jacardian_distance(char_series[char_1_index], char_series[char_2_index]))\n",
    "        data_structure_for_cosine_similarity.append(cosine_similarity(char_series[char_1_index], char_series[char_2_index]))\n",
    "\n",
    "# finally, find some way to present this data back. Either as a straight table or a heatmap.\n",
    "# so... yeah.... I can probably keep the rows the same as what they were... TECHNICALLY.... \n",
    "colz = char_series.index\n",
    "rowz = char_series.index\n",
    "\n",
    "\n",
    "for movChar in char_series_index:\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'char_series' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-15-de40192e3dd3>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mSeries\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnan\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mindex\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mchar_series\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'char_series' is not defined"
     ]
    }
   ],
   "source": [
    "pd.Series(np.nan,index = char_series.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Estimated number of clusters: 7\n",
      "Estimated number of noise points: 840\n",
      "Silhouette Coefficient: 0.108\n"
     ]
    }
   ],
   "source": [
    "db = DBSCAN(eps=0.05, min_samples=5).fit(X)\n",
    "core_samples_mask = np.zeros_like(db.labels_, dtype=bool)\n",
    "core_samples_mask[db.core_sample_indices_] = True\n",
    "labels = db.labels_\n",
    "\n",
    "# Number of clusters in labels, ignoring noise if present.\n",
    "n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)\n",
    "n_noise_ = list(labels).count(-1)\n",
    "\n",
    "print('Estimated number of clusters: %d' % n_clusters_)\n",
    "print('Estimated number of noise points: %d' % n_noise_)\n",
    "#print(\"Homogeneity: %0.3f\" % metrics.homogeneity_score(labels_true, labels))\n",
    "#print(\"Completeness: %0.3f\" % metrics.completeness_score(labels_true, labels))\n",
    "#print(\"V-measure: %0.3f\" % metrics.v_measure_score(labels_true, labels))\n",
    "#print(\"Adjusted Rand Index: %0.3f\"\n",
    "#      % metrics.adjusted_rand_score(labels_true, labels))\n",
    "#print(\"Adjusted Mutual Information: %0.3f\"\n",
    "#      % metrics.adjusted_mutual_info_score(labels_true, labels))\n",
    "print(\"Silhouette Coefficient: %0.3f\"\n",
    "      % metrics.silhouette_score(X, labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Estimated number of clusters: 7\n",
      "Estimated number of noise points: 840\n",
      "Silhouette Coefficient: 0.108\n"
     ]
    }
   ],
   "source": [
    "db2 = DBSCAN(eps=0.05, min_samples=7).fit(X)\n",
    "core_samples_mask2 = np.zeros_like(db.labels_, dtype=bool)\n",
    "core_samples_mask2[db.core_sample_indices_] = True\n",
    "labels2 = db.labels_\n",
    "\n",
    "# Number of clusters in labels, ignoring noise if present.\n",
    "n_clusters_2 = len(set(labels)) - (1 if -1 in labels else 0)\n",
    "n_noise_2 = list(labels).count(-1)\n",
    "\n",
    "print('Estimated number of clusters: %d' % n_clusters_)\n",
    "print('Estimated number of noise points: %d' % n_noise_)\n",
    "#print(\"Homogeneity: %0.3f\" % metrics.homogeneity_score(labels_true, labels))\n",
    "#print(\"Completeness: %0.3f\" % metrics.completeness_score(labels_true, labels))\n",
    "#print(\"V-measure: %0.3f\" % metrics.v_measure_score(labels_true, labels))\n",
    "#print(\"Adjusted Rand Index: %0.3f\"\n",
    "#      % metrics.adjusted_rand_score(labels_true, labels))\n",
    "#print(\"Adjusted Mutual Information: %0.3f\"\n",
    "#      % metrics.adjusted_mutual_info_score(labels_true, labels))\n",
    "print(\"Silhouette Coefficient: %0.3f\"\n",
    "      % metrics.silhouette_score(X, labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "labels_true must be 1D: shape is (3301, 6)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-55-e2a2d4ba959a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mhomogeneity_score\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mlabels\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\cluster\\supervised.py\u001b[0m in \u001b[0;36mhomogeneity_score\u001b[1;34m(labels_true, labels_pred)\u001b[0m\n\u001b[0;32m    388\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    389\u001b[0m     \"\"\"\n\u001b[1;32m--> 390\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mhomogeneity_completeness_v_measure\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabels_true\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabels_pred\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    391\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    392\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\cluster\\supervised.py\u001b[0m in \u001b[0;36mhomogeneity_completeness_v_measure\u001b[1;34m(labels_true, labels_pred)\u001b[0m\n\u001b[0;32m    294\u001b[0m     \u001b[0mv_measure_score\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    295\u001b[0m     \"\"\"\n\u001b[1;32m--> 296\u001b[1;33m     \u001b[0mlabels_true\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabels_pred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcheck_clusterings\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabels_true\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabels_pred\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    297\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    298\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabels_true\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\cluster\\supervised.py\u001b[0m in \u001b[0;36mcheck_clusterings\u001b[1;34m(labels_true, labels_pred)\u001b[0m\n\u001b[0;32m     51\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mlabels_true\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     52\u001b[0m         raise ValueError(\n\u001b[1;32m---> 53\u001b[1;33m             \"labels_true must be 1D: shape is %r\" % (labels_true.shape,))\n\u001b[0m\u001b[0;32m     54\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mlabels_pred\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     55\u001b[0m         raise ValueError(\n",
      "\u001b[1;31mValueError\u001b[0m: labels_true must be 1D: shape is (3301, 6)"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0,  0,  1, ...,  1,  1, -1], dtype=int64)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Estimated number of clusters: 80\n",
      "Estimated number of noise points: 591\n",
      "Silhouette Coefficient: -0.286\n",
      "Estimated number of clusters: 9\n",
      "Estimated number of noise points: 292\n",
      "Silhouette Coefficient: 0.164\n",
      "Estimated number of clusters: 3\n",
      "Estimated number of noise points: 146\n",
      "Silhouette Coefficient: 0.505\n",
      "Estimated number of clusters: 2\n",
      "Estimated number of noise points: 98\n",
      "Silhouette Coefficient: 0.709\n",
      "Estimated number of clusters: 2\n",
      "Estimated number of noise points: 66\n",
      "Silhouette Coefficient: 0.726\n",
      "Estimated number of clusters: 2\n",
      "Estimated number of noise points: 39\n",
      "Silhouette Coefficient: 0.745\n",
      "Estimated number of clusters: 2\n",
      "Estimated number of noise points: 22\n",
      "Silhouette Coefficient: 0.767\n",
      "Estimated number of clusters: 2\n",
      "Estimated number of noise points: 17\n",
      "Silhouette Coefficient: 0.780\n",
      "Estimated number of clusters: 1\n",
      "Estimated number of noise points: 12\n",
      "Silhouette Coefficient: 0.457\n"
     ]
    }
   ],
   "source": [
    "dbz = []\n",
    "shilz = []\n",
    "\n",
    "\n",
    "for minp, ep in zip([2,3,4,5,6,7,8,9,10],[.05,.075,.1,.125,.15,.175,.2,.225,.25]):\n",
    "    db = DBSCAN(eps=ep, min_samples=minp).fit(X)\n",
    "    core_samples_mask = np.zeros_like(db.labels_, dtype=bool)\n",
    "    core_samples_mask[db.core_sample_indices_] = True\n",
    "    labels = db.labels_\n",
    "    dbz.append(db)\n",
    "\n",
    "    # Number of clusters in labels, ignoring noise if present.\n",
    "    n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)\n",
    "    n_noise_ = list(labels).count(-1)\n",
    "\n",
    "    print('Estimated number of clusters: %d' % n_clusters_)\n",
    "    print('Estimated number of noise points: %d' % n_noise_)\n",
    "#print(\"Homogeneity: %0.3f\" % metrics.homogeneity_score(labels_true, labels))\n",
    "#print(\"Completeness: %0.3f\" % metrics.completeness_score(labels_true, labels))\n",
    "#print(\"V-measure: %0.3f\" % metrics.v_measure_score(labels_true, labels))\n",
    "#print(\"Adjusted Rand Index: %0.3f\"\n",
    "#      % metrics.adjusted_rand_score(labels_true, labels))\n",
    "#print(\"Adjusted Mutual Information: %0.3f\"\n",
    "#      % metrics.adjusted_mutual_info_score(labels_true, labels))\n",
    "    print(\"Silhouette Coefficient: %0.3f\"\n",
    "      % metrics.silhouette_score(X, labels))\n",
    "    shilz.append(metrics.silhouette_score(X, labels))\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shilz.index(max(shilz))\n",
    "dbz[7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "xlabz1 = pd.Series(dbz[7].labels_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ANTHONY____Alien-3_script',\n",
       " 'KYLE____Alien-3_script',\n",
       " 'RIPLEY____Aliens_script',\n",
       " 'HICKS____Aliens_script',\n",
       " 'HUDSON____Aliens_script',\n",
       " 'BURKE____Aliens_script',\n",
       " 'STANDARD____Alien_script',\n",
       " 'ROBY____Alien_script',\n",
       " 'HUNTER____Alien_script',\n",
       " 'BROUSSARD____Alien_script',\n",
       " 'ALI____Ali_script',\n",
       " 'BUNDINI____Ali_script',\n",
       " 'CASSIUS____Ali_script',\n",
       " 'HOWARD COSELL____Ali_script',\n",
       " 'MARY____All-About-Steve_script',\n",
       " 'HARTMAN____All-About-Steve_script',\n",
       " 'HOWARD____All-About-Steve_script',\n",
       " \"WILLIE____All-the-King's-Men_script\",\n",
       " \"JACK____All-the-King's-Men_script\",\n",
       " \"SADIE____All-the-King's-Men_script\",\n",
       " \"ANNE____All-the-King's-Men_script\",\n",
       " \"STANTON____All-the-King's-Men_script\",\n",
       " 'CARNBY____Alone-in-the-Dark_script',\n",
       " 'ALINE____Alone-in-the-Dark_script',\n",
       " 'MOZART____Amadeus_script',\n",
       " 'SALIERI____Amadeus_script',\n",
       " 'CONSTANZE____Amadeus_script',\n",
       " 'JOSEPH____Amadeus_script',\n",
       " 'OLD SALIERI____Amadeus_script',\n",
       " 'AMELIA____Amelia_script',\n",
       " 'GEORGE____Amelia_script',\n",
       " 'GENE____Amelia_script',\n",
       " \"AMELIA  (CONT'D)____Amelia_script\",\n",
       " 'FRED____Amelia_script',\n",
       " 'CLARA____American,-The_script',\n",
       " 'MATHILDE____American,-The_script',\n",
       " 'FATHER BENEDETTO____American,-The_script',\n",
       " 'LESTER____American-Beauty_script',\n",
       " 'RICKY____American-Beauty_script',\n",
       " 'JANE____American-Beauty_script',\n",
       " 'CAROLYN____American-Beauty_script',\n",
       " 'ANGELA____American-Beauty_script',\n",
       " 'FRANK____American-Gangster_script',\n",
       " 'TRUPO____American-Gangster_script',\n",
       " 'TOSCA____American-Gangster_script',\n",
       " 'CURT____American-Graffiti_script',\n",
       " 'TERRY____American-Graffiti_script',\n",
       " 'STEVE____American-Graffiti_script',\n",
       " 'DEBBIE____American-Graffiti_script',\n",
       " 'IRVING ROSENFELD____American-Hustle_script',\n",
       " 'RICHIE DIMASO____American-Hustle_script',\n",
       " 'CARMINE POLITO____American-Hustle_script',\n",
       " 'SYDNEY PROSSER____American-Hustle_script',\n",
       " 'KEVIN____American-Pie_script',\n",
       " 'JIM____American-Pie_script',\n",
       " 'OZ____American-Pie_script',\n",
       " 'HEATHER____American-Pie_script',\n",
       " 'SHEPHERD____American-President,-The_script',\n",
       " 'SYDNEY____American-President,-The_script',\n",
       " 'LEWIS____American-President,-The_script',\n",
       " 'LUCY____American-President,-The_script',\n",
       " 'RUMSON____American-President,-The_script',\n",
       " 'HARVEY____American-Splendor_script',\n",
       " 'REAL HARVEY____American-Splendor_script',\n",
       " 'TOBY____American-Splendor_script',\n",
       " 'LETTERMAN____American-Splendor_script',\n",
       " 'GEORGES____Amour_script',\n",
       " 'ANNE____Amour_script',\n",
       " 'EVA____Amour_script',\n",
       " 'SOLOIST____Amour_script',\n",
       " 'JENNY____An-Education_script',\n",
       " 'DAVID____An-Education_script',\n",
       " 'JACK____An-Education_script',\n",
       " 'DANNY____An-Education_script',\n",
       " 'HEADMISTRESS____An-Education_script',\n",
       " 'BEN____Analyze-That_script',\n",
       " 'LAURA____Analyze-That_script',\n",
       " 'JELLY____Analyze-That_script',\n",
       " 'PATTY____Analyze-That_script',\n",
       " 'BEN____Analyze-This_script',\n",
       " 'LAURA____Analyze-This_script',\n",
       " 'ANYA____Anastasia_script',\n",
       " 'DMITRI____Anastasia_script',\n",
       " 'VLADIMIR____Anastasia_script',\n",
       " 'SHARON____Angel-Eyes_script',\n",
       " 'ELANORA____Angel-Eyes_script',\n",
       " 'PINDELLA____Angel-Eyes_script',\n",
       " 'ROCHER____Angels-&-Demons_script',\n",
       " 'MORTATI____Angels-&-Demons_script',\n",
       " 'ANNA____Anna-Karenina_script',\n",
       " 'VRONSKY____Anna-Karenina_script',\n",
       " 'LEVIN____Anna-Karenina_script',\n",
       " 'KARENIN____Anna-Karenina_script',\n",
       " 'OBLONSKY____Anna-Karenina_script',\n",
       " 'ALVY____Annie-Hall_script',\n",
       " 'ANNIE____Annie-Hall_script',\n",
       " 'ROB____Annie-Hall_script',\n",
       " 'TONY____Annie-Hall_script',\n",
       " 'OXFORD____Anonymous_script']"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = pd.read_csv(os.getcwd() + '/top5_chars.csv')\n",
    "x['clustering_labels'] = xlabz1\n",
    "x_1 =  x[x['clustering_labels']==1]['Unnamed: 0']\n",
    "x_0 = x[x['clustering_labels'] == 0]['Unnamed: 0']\n",
    "#[['clustering_labels','Unnamed: 0']]\n",
    "list(x_1)[61:160]"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "x_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['__class__',\n",
       " '__delattr__',\n",
       " '__dict__',\n",
       " '__dir__',\n",
       " '__doc__',\n",
       " '__eq__',\n",
       " '__format__',\n",
       " '__ge__',\n",
       " '__getattribute__',\n",
       " '__getstate__',\n",
       " '__gt__',\n",
       " '__hash__',\n",
       " '__init__',\n",
       " '__init_subclass__',\n",
       " '__le__',\n",
       " '__lt__',\n",
       " '__module__',\n",
       " '__ne__',\n",
       " '__new__',\n",
       " '__reduce__',\n",
       " '__reduce_ex__',\n",
       " '__repr__',\n",
       " '__setattr__',\n",
       " '__setstate__',\n",
       " '__sizeof__',\n",
       " '__str__',\n",
       " '__subclasshook__',\n",
       " '__weakref__',\n",
       " '_estimator_type',\n",
       " '_get_param_names',\n",
       " 'algorithm',\n",
       " 'components_',\n",
       " 'core_sample_indices_',\n",
       " 'eps',\n",
       " 'fit',\n",
       " 'fit_predict',\n",
       " 'get_params',\n",
       " 'labels_',\n",
       " 'leaf_size',\n",
       " 'metric',\n",
       " 'metric_params',\n",
       " 'min_samples',\n",
       " 'n_jobs',\n",
       " 'p',\n",
       " 'set_params']"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dir(db)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "core_sample_indices_ : array, shape = [n_core_samples]\n",
    "Indices of core samples.\n",
    "\n",
    "components_ : array, shape = [n_core_samples, n_features]\n",
    "Copy of each core sample found by training.\n",
    "\n",
    "labels_ : array, shape = [n_samples]\n",
    "Cluster labels for each point in the dataset given to fit(). Noisy samples are given the labe"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}