tutorials/K-means/039-kmeans_exploredata.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n",
      "-----------\n",
      "          country   region   tfr  contraception  educationMale  \\\n",
      "0     Afghanistan     Asia  6.90            NaN            NaN   \n",
      "1         Albania   Europe  2.60            NaN            NaN   \n",
      "2         Algeria   Africa  3.81           52.0           11.1   \n",
      "3  American.Samoa     Asia   NaN            NaN            NaN   \n",
      "4         Andorra   Europe   NaN            NaN            NaN   \n",
      "5          Angola   Africa  6.69            NaN            NaN   \n",
      "6         Antigua  America   NaN           53.0            NaN   \n",
      "7       Argentina  America  2.62            NaN            NaN   \n",
      "8         Armenia   Europe  1.70           22.0            NaN   \n",
      "9       Australia  Oceania  1.89           76.0           16.3   \n",
      "\n",
      "   educationFemale  lifeMale  lifeFemale  infantMortality  GDPperCapita  \\\n",
      "0              NaN      45.0        46.0            154.0        2848.0   \n",
      "1              NaN      68.0        74.0             32.0         863.0   \n",
      "2              9.9      67.5        70.3             44.0        1531.0   \n",
      "3              NaN      68.0        73.0             11.0           NaN   \n",
      "4              NaN       NaN         NaN              NaN           NaN   \n",
      "5              NaN      44.9        48.1            124.0         355.0   \n",
      "6              NaN       NaN         NaN             24.0        6966.0   \n",
      "7              NaN      69.6        76.8             22.0        8055.0   \n",
      "8              NaN      67.2        74.0             25.0         354.0   \n",
      "9             16.1      75.4        81.2              6.0       20046.0   \n",
      "\n",
      "   economicActivityMale  economicActivityFemale  illiteracyMale  \\\n",
      "0                  87.5                     7.2          52.800   \n",
      "1                   NaN                     NaN             NaN   \n",
      "2                  76.4                     7.8          26.100   \n",
      "3                  58.8                    42.4           0.264   \n",
      "4                   NaN                     NaN             NaN   \n",
      "5                   NaN                     NaN             NaN   \n",
      "6                  74.4                    56.2             NaN   \n",
      "7                  76.2                    41.3           3.800   \n",
      "8                  65.0                    52.0           0.300   \n",
      "9                  74.0                    53.8             NaN   \n",
      "\n",
      "   illiteracyFemale  \n",
      "0             85.00  \n",
      "1               NaN  \n",
      "2             51.00  \n",
      "3              0.36  \n",
      "4               NaN  \n",
      "5               NaN  \n",
      "6               NaN  \n",
      "7              3.80  \n",
      "8              0.50  \n",
      "9               NaN  \n",
      "-----------\n",
      "Individual columns - Python data types\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('country', str),\n",
       " ('region', str),\n",
       " ('tfr', numpy.float64),\n",
       " ('contraception', numpy.float64),\n",
       " ('educationMale', numpy.float64),\n",
       " ('educationFemale', numpy.float64),\n",
       " ('lifeMale', numpy.float64),\n",
       " ('lifeFemale', numpy.float64),\n",
       " ('infantMortality', numpy.float64),\n",
       " ('GDPperCapita', numpy.float64),\n",
       " ('economicActivityMale', numpy.float64),\n",
       " ('economicActivityFemale', numpy.float64),\n",
       " ('illiteracyMale', numpy.float64),\n",
       " ('illiteracyFemale', numpy.float64)]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%pylab inline\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "df = pd.read_csv('/home/sophie/projects/LearnDataScience-master/datasets/UN.csv')\n",
    "\n",
    "print('-----------')\n",
    "# raw column information plus summary header\n",
    "print(df[0:10])\n",
    "print('-----------')\n",
    "# look at the types of each column explicitly\n",
    "print('Individual columns - Python data types')\n",
    "[(x, type(df[x][0])) for x in df.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'str'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('country', str),\n",
       " ('region', str),\n",
       " ('tfr', numpy.float64),\n",
       " ('contraception', numpy.float64),\n",
       " ('educationMale', numpy.float64),\n",
       " ('educationFemale', numpy.float64),\n",
       " ('lifeMale', numpy.float64),\n",
       " ('lifeFemale', numpy.float64),\n",
       " ('infantMortality', numpy.float64),\n",
       " ('GDPperCapita', numpy.float64),\n",
       " ('economicActivityMale', numpy.float64),\n",
       " ('economicActivityFemale', numpy.float64),\n",
       " ('illiteracyMale', numpy.float64),\n",
       " ('illiteracyFemale', numpy.float64)]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The key point is that in the for loop below we are looking at the first VALUE in each of the columns \n",
    "# to see what datatypes they are. df.dtypes will simply say \"object\".\n",
    "print (type(df['country'][0]))\n",
    "\n",
    "# make a list which looks at the data type from\n",
    "[(x, type(df[x][0])) for x in df.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "country                    object\n",
      "region                     object\n",
      "tfr                       float64\n",
      "contraception             float64\n",
      "educationMale             float64\n",
      "educationFemale           float64\n",
      "lifeMale                  float64\n",
      "lifeFemale                float64\n",
      "infantMortality           float64\n",
      "GDPperCapita              float64\n",
      "economicActivityMale      float64\n",
      "economicActivityFemale    float64\n",
      "illiteracyMale            float64\n",
      "illiteracyFemale          float64\n",
      "dtype: object\n"
     ]
    }
   ],
   "source": [
    "# This prints out the format of the COLUMN. the first two are Series objects.\n",
    "# The others are numpy float arrays.\n",
    "print (df.dtypes)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "country                   207\n",
       "region                    207\n",
       "tfr                       197\n",
       "contraception             144\n",
       "educationMale              76\n",
       "educationFemale            76\n",
       "lifeMale                  196\n",
       "lifeFemale                196\n",
       "infantMortality           201\n",
       "GDPperCapita              197\n",
       "economicActivityMale      165\n",
       "economicActivityFemale    165\n",
       "illiteracyMale            160\n",
       "illiteracyFemale          160\n",
       "dtype: int64"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# How many non null values are there in each column\n",
    "df.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### What does the above tell us?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 14 columns, 207 countries\n",
    "- We want to focus on columns with the least null values - tfr, lifeMale,lifeFemale,GDP and InfantMortality\n",
    "- We suspect a clustering of lifeMale, lifeFemale and infantMortality according to GDP - will use K-means to analyse \n",
    "in detail\n",
    "- We don't know how many clusters there might be\n",
    "- Will use analytics too, to help decide the right number of clusters might be (possible 1 to 10)\n",
    "- Each field will be converted to a scientific float needed by the numerical algorithms (are numpy floats not right?)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Methods"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We want to find out how many natural clusters there are. This is tested by appling k-means clustering using a different number of clusters (from 1, to 10). When we have more clusters, they will be more tightly packed and least squares error will be small. Less clusters and each cluster will have a bigger spread. \n",
    "\n",
    "However, each successive increase in K will not give the same drop in tightness. \n",
    "\n",
    "The point at which the improvement starts to level off is \"the elbow\". We will use the value of k at this point."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**steps**\n",
    "- generate elbow curve, decide on k, run clustering over the data\n",
    "- plot the clusters to visualise exactly how the clusters appear\n",
    "- interpret the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline\n",
    "# using the code from \n",
    "# http://stackoverflow.com/questions/6645895/calculating-the-percentage-of-variance-measure-for-k-means\n",
    "# we will generate plots from the UN dataset transformed into 4 numeric columns\n",
    "# lifeMale, lifeFemale, infantMortality and GDPperCapita\n",
    "\n",
    "fName = ('/home/sophie/projects/LearnDataScience-master/datasets/UN4col.csv')\n",
    "fp = open(fName)\n",
    "X = np.loadtxt(fp)\n",
    "fp.close()\n",
    "         "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(188, 4)\n",
      "<class 'numpy.ndarray'>\n",
      "2\n",
      "752\n"
     ]
    }
   ],
   "source": [
    "# A little query on the numpy array.\n",
    "print (X.shape)\n",
    "print (type(X))\n",
    "print (X.ndim)\n",
    "print (X.size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as mp\n",
    "from scipy.cluster.vq import kmeans,vq\n",
    "from scipy.spatial.distance import cdist\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "#### cluster into K=1..10 clusters #####\n",
    "# K, KM, centroids, D_k, cIDx, dist, avgWithinSS = kmeans.run_kmeans(X,10)\n",
    "\n",
    "K = range(1,10)\n",
    "\n",
    "# apply kmeans 1 to 10. kmeans(obs, k_or_guess[, iter, thresh, ...]). \n",
    "# returns a codebook(A k by N array of k centroids) and \n",
    "# distortion (a float describing the distortion between the observations passed and the centroids generated)\n",
    "# KM is a list of codebooks and distortion for different numbers of clusters, from 1 to 10.\n",
    "KM = [kmeans(X,k) for k in K]\n",
    "\n",
    "# centroids is a list of codebooks for each K. cent is the codebook, a numpy array, \n",
    "# for 1 cluster, 2 clusters, 3 clusters etc..\n",
    "centroids = [cent for (cent,var) in KM]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What information is in a codebook?\n",
    "It maps centroids to codes and vice versa. The format is [n_clusters, n_features]. See below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 4)\n",
      "(2, 4)\n",
      "(3, 4)\n",
      "(4, 4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[(1, 4), (2, 4), (3, 4), (4, 4), (5, 4), (6, 4), (7, 4), (8, 4), (9, 4)]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This is one way to look at each element in a list\n",
    "print(centroids[0].shape)\n",
    "print(centroids[1].shape)\n",
    "print(centroids[2].shape)\n",
    "print(centroids[3].shape)\n",
    "\n",
    "# This is much better\n",
    "[s.shape for s in centroids]  # list the shape of each code book."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# cdist computes the euclidean distance between two points.\n",
    "# Every value in the array is tested against each code book. \n",
    "# cdist(XA, XB, \"metric\"). \n",
    "# Where XA = mA by n array of mA original observations in an n-dimensional space\n",
    "# and XB = An mB by n array of mB(no of clusters) original observations in an n-dimensional space. \n",
    "# Both must have the same number of columns.\n",
    "D_k = [cdist(X, cent, 'euclidean') for cent in centroids]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Don't quite understand what cdist does. I think D_k is a list matrices, each of which contain a vector for each value of X, for each cluster set. \n",
    "We end up with a list of mA by mB distance matrices. i.e. number of rows in X by number of clusters. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What is the difference between a centroid and a cluster center?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(188, 4)\n",
      "9\n",
      "<class 'list'>\n",
      "[ 2164.54439528]\n"
     ]
    }
   ],
   "source": [
    "print(X.shape)\n",
    "print(len(D_k))\n",
    "\n",
    "# if D_k is a class object of list, what attributes does that have?\n",
    "print(type(D_k))\n",
    "\n",
    "# This is how to access individual bits of the matrix.\n",
    "print(min(D_k[0][3:6])) # Selecting 3:6 from the first matrix. i.e. the matrix made for only one cluster."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "188"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cIdx = [np.argmin(D, axis=1) for D in D_k]\n",
    "dist = [np.min(D, axis=1) for D in D_k]\n",
    "avgWithinSS = [sum(d)/X.shape[0] for d in dist]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What is the difference between the argmin (cIdx) and min(dist)? cIdx is a list of indices for the minimum values. dist is those values.\n",
    "\n",
    "cIdx = for each distance matrix we have an array of indices of the min values along an axis.          \n",
    "dist = for each distance matrix we have an array of the min values along an axis            \n",
    "avgWithinSS = for each of the matrices sum(min values)/188 (what is the average min value in each of the different cluster numbers i.e. with 1 cluster the average min value is avgWithinSS[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(188, 1),\n",
       " (188, 2),\n",
       " (188, 3),\n",
       " (188, 4),\n",
       " (188, 5),\n",
       " (188, 6),\n",
       " (188, 7),\n",
       " (188, 8),\n",
       " (188, 9)]"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print ([D.shape for D in D_k]) # each one of these has a array of indices which have the min values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "       0, 0, 0, 0]), array([1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0,\n",
      "       1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1,\n",
      "       1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1,\n",
      "       1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1,\n",
      "       1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1,\n",
      "       1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "       1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1,\n",
      "       1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1,\n",
      "       1, 1, 1, 1]), array([0, 0, 0, 0, 1, 0, 2, 2, 0, 1, 1, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1,\n",
      "       0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0,\n",
      "       0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 1, 0, 0, 0, 2, 0, 1, 1, 0, 0, 0, 0,\n",
      "       0, 0, 2, 0, 2, 0, 0, 0, 1, 1, 1, 1, 0, 0, 2, 0, 0, 0, 0, 0, 1, 1, 0,\n",
      "       0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0,\n",
      "       0, 0, 0, 0, 0, 2, 1, 1, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,\n",
      "       1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 1, 0, 0, 0, 1, 0, 0, 0,\n",
      "       0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 0, 0, 0, 0, 0,\n",
      "       0, 0, 0, 0])]\n"
     ]
    }
   ],
   "source": [
    "print(cIdx[0:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 3044.71049474  5027.61602297  4359.59802141]\n",
      "[ 980.43054626  425.17523451  343.95436068]\n"
     ]
    }
   ],
   "source": [
    "print(dist[0][0:3]) # first three points in the first matrix for one cluster. \n",
    "print(dist[6][0:3]) \n",
    "# As you increase the number of clusters these distances decrease as the clusters become more compact."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n"
     ]
    }
   ],
   "source": [
    "print(len(dist))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[6534.9809626620172, 2790.2101193300145, 1890.9166153060171, 1438.7793254224123, 1122.2625658294346, 972.53807558001813, 740.45942949866037, 645.91915410445324, 648.92982126597178]\n"
     ]
    }
   ],
   "source": [
    "print(avgWithinSS)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}