kmeans.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## K-means\n",
    "\n",
    "* test correctness of NLP vectorization\n",
    "* test logic of kmeans\n",
    "* test kmeans on text data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pyspark as ps\n",
    "import json\n",
    "\n",
    "# run it single threaded\n",
    "sc = ps.SparkContext('local')\n",
    "\n",
    "test_rdd = sc.textFile('file:///Users/jonathandinu/spark-ds-applications/data/dummy_clusters.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "test_rows = test_rdd.map(lambda line: np.array(map(float, line.split(','))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x112ca4210>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAEBCAYAAACHTjUfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X9sXOWZL/DveBI7ttOAjXPvrMSWQkliddAK6V7tUjrR\nGoWSRbgbhd5565UucMtSsZdIsU22DWGTBtdcKE1SJ7kLKiokLKmI+3oDbS+05Icp3RkCUfdKV71M\nm5KyCG4lshBmbMeOEzvjc/+wT5iMZ+bMj3fOnOf4+5FeKTOZmfPNceaZ4+e8552AZVkgIiJ/qKt1\nACIiModFnYjIR1jUiYh8hEWdiMhHWNSJiHyERZ2IyEcWlftEpdRqALsA/Epr/c25+24FsH3uIdu1\n1q9VHpGIiIpVyZF6A4DH7RtKqToAfQBumxuPKKUClcUjIqJSlF3UtdbHACQz7loB4B2t9aTWehLA\nuwCurzAfERGVoOz2Sw6tAEaUUgNzt0cBXAXglMFtEBFRASaL+icArgTwAIAAgKcAnDH4+kRE5KDS\nop7ZM38XwMqM2yu01n/I98Th4WEuOkNEVIY1a9bkPV9ZyeyXzQBuBxBSSi3TWt+vlOoDcHTuIY9U\nEoyIiOZzOiAO1GqVxuHhYUtCUQ8EAh2WZb1e6xxOJOSUkBFgTtOY0yyn2smLj4iIfIRF3YGET25A\nRk4JGQHmNI053cWiTkTkIyzqDgKBQEetMxRDQk4JGQHmNI053cWiTkTkIyzqDqT02STklJARYE7T\nmNNdLOpERD7Cou5ASp9NQk4JGQHmNI053cWiTkTkIyzqDqT02STklJARYE7TmNNdLOpERD7Cou5A\nSp9NQk4JGQHmNI053cWiTkTkIyzqDqT02STklJARYE7TmNNdLOpERD7Cou5ASp9NQk4JGQHmNI05\n3cWiTkTkIyzqDqT02STklJARYE7TmNNdLOpERD7Cou5ASp9NQk4JGQHmNI053cWiTkTkIyzqDqT0\n2STklJARYE7TmNNdLOpERD7Cou5ASp9NQk4JGQHmNI053cWiTkTkI4tqHcDrpPTZJOSUkBFgTtOK\nyZlKpdoSiWA3AITD6T0tLS1nqh4si5T96YRFnYhqKpVKtR0+vPj13t6mMAAMDJxbv3ZtqqMWhd0P\n2H5xIKXPJiGnhIwAc5rmlDORCHb39jaFT5+uw+nTdejtbQrbR+1ukrI/nfBInYhK5oV2CeXGI3UH\nUvpsEnJKyAgwpxO7XRKNLt0ajS7devjw4tdTqVRbvsc75QyH03sGBs4lQqEZhEIzGBg4lwiH03uM\nB3cg5efuhEfqRFSSzHYJAPT2NoWHhsa7IxFsK+f1Wlpazqxdm+oYGho3duS/kH+T4JG6Ayl9Ngk5\nJWQEmNO0YnK2tLSciUSWbYtElm0zUdBL+U2ilJwS8EidiEoSDqf3PPnkxH+JxRa1A8Dq1RdP1qJd\nko/p3ySkYVF3IKXPJiGnhIwAc9oKtTCmpxEYHGwAANx008VALXOaIiWnE7ZfiGieQi2MRCLY3dPT\nvMqegtjT07yqFlMQ8/HKiddaYVF3IKXPJiGnhIwAcwJm545XM2cqlWqLx8f64/GxfvtDZ/bE63TH\n0ND4o0ND44+uXTtd1IVMUn7uTth+IaKSzB0Jr8+4AtTYkXAps1YKXYk6e+J1YfTQ57Esy+iIRqPP\nRaPRN6PR6C+j0eg9+R537Ngxy/S2OTg4zIxkMtl28ODZt0OhtBUKpa2DB8++nUwm2zL/PhYb7Y/F\nRvsz76/mNrNHLDbaHwqlLcCyAMsKhdJWLDbaX852Tf9bqjmcamc1jtQtAF/TWn9QhdcmIhc4zR0v\n9Ug41xG4fV8waDUGg8DUVN1fuD1rxY/rzlSr/VLwbLgkgUCgwxJwVlxCTgkZAea0mWhhzBbNd54M\nBr9wy8aNzcuBS4XzzsOHF79oF9PNmydx+nRpZSO7DbR378TH4XD6QCmv4cfpj9U4UXoWwAtKqf+l\nlLq+Cq9PRALYR8EnTvyZ2rixefnlJ13rns08EfvEE42YnAxg8+ZJFDtrZe6E6J379o1/3NV1Ad/+\nduPyw4cXv1jMhUa+Vq2+TzQavTEajb5UqC8EoMO+DaCDt3mbt71zOxKJrLN7zZFIZF2pz9+3L/Z8\nKJS27rtv0srufb/yynu/yb7vvvsmrfb2aSsWG4nt2xd7PhKJrHPaXr6+erH/3nx9fC/s/3y3nXrq\nef+i0hGNRtuj0ajO9/c8UcrB4d1R6knLXMMuuK2taWvHjgkr67VWZr7+wMC41d4+XfJ2TJws9duJ\nUuMbjEajg9Fo9FfRaPTlaDR6TbnBvDIyPy29PCTklJCROWeHqWJpF+729mnr5z8f/ej48ZGdduG0\ni+nx4yM7T5wY2VlOUTXx4SPt5+767BetdZfp1yQibyo0r9yeQfPYY8e/v2LFn71fzgwap3nr1Vjh\nUTyvftpwcHDUbhRzBGzyKLncDAtxONVOLhNARPMUc6l9tb6Gzr70P5Goe6mvb0l4agro7JzCiRPB\n8KlTdQ9V+vp+x6LuQMp6EBJySsgIMCdg5ksm7OK8f3/8+WKnGaZSqZVvvRX87aFDi7d+4xvNkY0b\nz2Pr1km8/HI9Bgcb8P77dXdXa8qilJ+7ExZ1IrpMsV8yEQ6nDwwMTFz4dF75xAX74p/M13j44Zvv\nKuaLKlKpVNuxY4vi9967dPngYAP+9m+n8P77dfje9xph/zawcWPzci+tCOlFLOoOLAFXFgIyckrI\nCDBnsW2VRCJ4V19fY0Nn5xQ6O6fQ19fYkEgE7yr2NbJXWEwkgt2ZFynt2rUEn//8TDX+iTlJ+bk7\n4SqNRFS2kZE6PPPMEgBAKFR8Ac615sp116VfzX7c5z6XPrN798QnPT3Nq+YeN+8qU1PfR+qb7zX1\n6hlcrwwImbsqIaeEjMxZ/KyTZDLZNjg4dtJ+3ODg2MnMOeiZr/Hii6On3nhj5M1YbCSWTCZX5poH\nf/z4yM7M52g99lEymVxZ6OIgUzNkkslk2/79H74nYaZNLVZpJCLBSpn7vXgxrK6uC5f+nOs1/v3f\nT90wNbXy9q9+den1ALBnz8Rvr7125vns10qnA5NzM27mbTfffHZTC3IlEsHuLVv+w+f8sLAXi7oD\nS0ifTUJOCRkB5gSKuzAokQh2b9jQ3G4Xwldfvdj+wx9OvASMvW4X5EgE2+Lx62PRaHOD/bju7ubg\ngQNnv7Z798Tvs9sqpr7cwjetlDLwRCkRVay1dQb3338B0ehnIoVmzNheeaW+6dprZ14u9SvnsuX5\nPtIDxczeKeJ1ZH6vqVf7Ql4ZWOD91YWWkTmLH5n97J6ec/NWYrTXirn33nv/649/PHbR7lfv2DEx\ntxpj6d9SlC9HZs+93HVrMlel9Go/3bLYUyeiKrm89251DA42RHI9bt++fX/cuXPnl/bvH3/t8OHF\nTc8+W4/t288bOxKe37IZK+t14vH4aCSyrKK1aDzBq582HBwcckaxa8W4cSRcrTVjvLIWjVPtDFiW\n5Vz5q2B4eNhas2aNb772jmih89JRbDWyxONj/dHo0q32Cd9QaAZDQ+OPOh3dm+ZUO9l+ccDvqzRH\nQkaAOcuVb+ZKLXKWM4vGa/uzXJz9QkRUBCkzZHik7kDKJ7eEnBIyAsxpml9ySvlCDhZ1IlpQKum3\nm7o4qprYfnEgZY1lCTklZASY0zQv5Sy0rLCXclaCRZ2IxMpevtfp8dX6tiYvYfvFgV/6gV4gISPA\nnKZVK2eu5XvXrk2VtdwAIGd/OuGROhGJVM5Rt5QZLJVgUXcgpc8mIaeEjABzmualnIW+UNtLOSvB\n9gsRiTR31L0+o/1S1FG3hBkslWBRdyClzyYhp4SMAHOaVq2cpueNS9mfTljUiUgsvx91l4M9dQdS\n+mwSckrICDCnaczpLhZ1IiIfYVF3IKXPJiGnhIwAc5rGnO5iUSci8hEWdQdS+mwSckrICDCnaczp\nLhZ1IiIfYVF3IKXPJiGnhIwAc5rGnO5iUSci8hEWdQdS+mwSckrICDCnaczpLhZ1IiIfMb5MgFLq\nVgDb525u11q/ZnobbpLSZ5OQU0JGgDlNY053GT1SV0rVAegDcNvceEQpFTC5DSIiys90+2UFgHe0\n1pNa60kA7wK43vA2XCWlzyYhp4SMAHOaxpzuMt1+aQUwopQamLs9CuAqAKcMb4eIiHIwXdQ/AXAl\ngAcABAA8BaDs9Y29QEqfTUJOCRkB5jSNOd1luv3yLoCVGbdXaK3/kO/Bmb/uBAKBDt7mbd7mbd52\nvl1IwLKsYh5XNKXUbQC+PXezT2t9NNfjhoeHrTVr1nj+JGogEOiQ8AkuIaeEjABzmsacZjnVTuNT\nGrXWRwAcMf26RETkjBcfOZDwyQ3IyCkhI8CcpjGnu1jUiYh8hEXdQbEnJ2pNQk4JGQHmNI053cWi\nTkTkIyzqDqT02STklJARYE7TmNNdLOpERD7Cou5ASp9NQk4JGQHmNI053cWiTkTkIyzqDqT02STk\nlJARYE7TmNNdLOpERD7Cou5ASp9NQk4JGQHmNI053cWiTkTkI8YX9PIbKX22audMpVJtiUSwGwDC\n4fSelpaWktfJ5740iznNkpLTCYs6OUqlUm2HDy9+vbe3KQwAAwPn1q9dm+oop7ATUXWx/eJASp+t\nmjkTiWB3b29T+PTpOpw+XYfe3qawfdReCu5Ls5jTLCk5nbCoExH5CIu6Ayl9tmrmDIfTewYGziVC\noRmEQjMYGDiXCIfTe0p9He5Ls5jTLCk5nbCnTo5aWlrOrF2b6hgaGq/oRCkRVR+LuoNAINCRTCbf\nrnTmR6lKnW1S7e9XbGlpOROJYFslryHlOyCZ0yzmdBeLuoNIJHKF2zM/ypltEolErojHx/oBHkkT\nLWiWZdVkHDt2zKrVtksZsdhofyiUtgDLAiwrFEpbsdhov5e2mUwm2w4ePPt2KJS2QqG0dfDg2beT\nyWRbrfcdBweH+eFUO3mitEpSqVRbPD7WH4+P9adSqbZqbsuecjg1BXR2TuHEiWD41Km6h6q5TSLy\nJhZ1B1u23PGvpc78sNsn0ejSrdHo0q2HDy9+vZTCnj3b5MknJ04Gg1ZjoQ+IK6+cwZYt5/Hyy/UY\nHGzABx/U/fdUKrWy1H9vNUmZB8ycZjGny7z6K4RXBoCOZDLZFouN9sdio/3FtDXKbdlkbWdlLDba\nf+LEyM7BwbGThVoryWSy7Wc/O5PK3ubPfz76kZfaMAA6ap2BOZlTek6n2skTpQ6subPhlc78cJL7\n5Oh0RyIR7O7paV51+vTsL1W9vU3hoaHx7sw8LS0tZ9raRp8FsCnzNY8cWbz8qqush6amxiaB2p9A\ntYTMLGBOs5jTXWy/VEE4nD4wMDBx4dOWzcSFcDh9oNBzKr0Uv7195rt79058bG9z06bzePPNIN5/\nv+7ucttARCQPi7qDUvps9snRRKLu2b6+xobOzil0dk6hr6+xIZEI3lXoufX1VmNX1wXcd995tLbO\nXLq/2Ks5W1tbb7j11otf+dGPzp77znfO4Z//eRG2bz//8caNzcvL/aAo5WRvMY+V0rNkTrOY011s\nvxiS2T7p6rqAkZE6PPPMEgBAKDTj+Nz33lvUOTjYAADYtm0Sn/3szEm7XVLM1Zxz8+n/sbe3qQkA\n9u6d+Li11RoC8ECl/x6g8Fx5ruJI5CFebfZLG5knR1tb09bAwLhln9wcHBw7eeLEyM58J1pznVg9\nfnxkZ7nbt1/j179OPZl5knVwcOxksSdOc5/sHYkVm7/ac/k5OBbq4InSGkgm6/D00w0YGjobDwZx\n4sMP6+5Yt+4zm4Dij2LT6cAk8OlyAfX1VmM6PXt/sSc8U6mAqq+3Rrq6LgAAFi+GVcm/69Ch+sgf\n/5h+nUfhRB7m1U8brwwUOc0p31WdxRzF5nuufX97+/RlR/65pjVGIpF1Wo99ZD9mx44Ja/PmCavc\nI+jsTDt2TFitremS8pe7L2s9mJM5vTx4pO6S/L3vsbKfG4+P9ff2NoU7O6fwxBONmJoC7rxz9orR\n665LP/Tnf46/t18jHo+P7to183xX14VN4+MBPP74Etx99wUD/56zLx06VB95/PElSCbrcp4f4CqO\nRB7i1U8bv4xK1mWxj/Lvu2/Sam+ftnbsmLh0tK712LwLi7K3dejQ2O+cLlyqZn4ODg7zw6l2Biyr\nojZr2YaHh601a9YEarJxl5X7pc32rJK+viXhvr5JdHc3w74IKRSawdDQ+KORyLJt2c/J3BYwOwe+\n1G2byE9E5jnVTrZfHBSzxrJT0St3LXK7rXH11TPddXXWrQBucsqZa1uVXg1rYi11QM561cxpFnO6\nixcfVajSxbuKtWgR3ti9e+L39kVIe/dOfOx0lWox3FxNkohc4NW+kJRRzTna2f3sn/1s9J1f/GL0\nTE/POau9fdo6dGjsd8eP55//Xurrs1/OweH94frsF6XUcwBWATgP4Dmt9T+Z3sZCkbkeDAC89tri\nFYODDTh9ug6trTP44IO69g0bmtuB8q7izH793t6m8Isvnn3oi1/8dFYNEclSjfaLBeBrWutb/FDQ\nndaDKHZtFtPuvHN2mmO567oAudebGRkJ3F2tNoyUtTWY0yzmdFe1TpQuiFktQHXnaM99YKy311RZ\nvfriyZtuuhjo6WletXRpZbOWcq03EwwCDz7YtPyHPzzXXe2lhomoOsou6kqpLwP4VtbdmwCcBfCC\nUioJoFdr/YcK8tWcVcTZcFOzQ3K9bvYHBgAMDY1319dbjTfddLGzp6d5FYCSf0PIXqf9iScaEY3O\nLkRWLcXsSy9gTrOY011lF3Wt9VEAR3P81UYAUErdCGAHgPXlboNyf2DYt1esSH230G8Ipc4vDwZL\n/3AgIo+p1hnaaDTaHo1GdaEzuMhYawFAhxdv2/d5JU+B2z2ZtyORyLr9+z98z57Zsn//h+9FIpF1\n+f7+hRc+SR08+NaP7dkvVcrb46H9lfd29s++1nm4P7k/M2+7fkWpUmoQwJ9gtg2zQWv9fq7HSbmi\nVMoFCdk54/Gx/mh06dZCV6C6faWo1H3pVcxplpScrl9RqrXuMv2ateTlH3JmUU4mkyW3TKp1LiAf\nL+/LTMxpFnO6i1eUCuV0JWutploSUW2xqDvw6txVpy+qnp05M90xNDT+6NDQ+KNr107X/IstvLov\nszGnWczpLi7o5VNcWbF09j7bty92TSqVepv7jEQqdBa1msMva7/UahRat4VrupjdnxwcXhpOtZPt\nF6EKtVecWjM0H/cZ+QWLugMv99lmZ68s2xaJLNvW2tp6Q63zOPHyvpRIyv5kTnexqPsQZ76UjvuM\n/IJfZ+dTPFFaOu4zkoBfZ7dAuX1hkR9wn5EfsP3iQEqfTUJOCRkB5jSNOd3FI3XyLbZTaCFiUXdg\nCVkPQkJONzPayyjYXzBSytf9SdiXAHOaJiWnE7ZfyJec5p2nUqm2eHysPx4f6zf99X3VfG0iJyzq\nDqT02STk9EpGp8XQKsnp9NomeWV/OmFOd7GoU83kO6I1caRbaN55Na8e5ZWpVGvsqTuQ0meTkDMz\nY76eNwCU2wvPVOoXggeDVmOunF7GnGZJyenIq4vScPh7xGKj/aFQ2gIsC7CsUChtxWKj/fnuz/Ua\nyWSyzX5OMplcmfHnggtxJZPJtsHBsZP24l0DA+PWoUNjvzOxgBcXBuOo9uCCXhWS0meTkNNkxvm9\n60W/+cY3morqY7e0tJy59tqZl7u6LqCzcwr9/Y3YsKG53W6TVJLTzXXsJfzMAeZ0G9svVBNzPe/1\nGW2WSz3vfPdnyuxdA0Bvb3NDZ+cUnnlmCXp7m8JDQ+PdmVeHZs9Zn5oKTg4ONiDzO1xtkUjkinh8\nrN9+bDmtH16ZSrXCou7AEtJnq3XOYi70ycxYqOddSi+82Gzz+/TTd+b68EilUm0bNvzif0SjlfX0\n3VDrn3mxmNNlXu0LccgZtegjz9/m2Pn29umcXxgSi428matPn7snPxIrtqfPwVGL4VQ7eaTuIBAI\ndFgCPsFrmXN+K2R++8N0xhxH+geuvvrcXXN/3tPS0nImlUq1HTmyKP7WW4tWZT8/GLQaM36zOHD4\n8OIXe3ubwl1dF0zEcwX/b5olJacTFnUSK7t3HQ6n9iQSwe5EItht/7mnp3nV1BSwZct57Nq1BACw\nd+/EJx9+WHfHhg3N7QCwb9/4/b29TctPn67D8883YNu2STzxxOwMR66rTtKwqDuQ8sldy5yFTnpm\nKjdjKpVqO3Wq7qGpKfxFMIgT7e0z383ucWf3zXfvnohee236bFfXBYyPB/CDH9Sjq+sCvvCFNP70\nT9On1q9fdpP9m8WRI4uX26+TTNbh6acbMDR0Ng4EXvfyQmD8v2mWlJxOWNSpYqVe6FOKVCrV9stf\nLop98EFd+9zRc2T37onO225L/XUiEbzUbjl5su6hzBbQI480rtq8eRKDgw0AgM2bJ9HQYOGqq6yT\ndXWBNwDcZG/j1VcXY+/eiY83bmxeDgDbt59PhMMz6+1/gxurPXJFSTLGq81+rwwAHbXO4Jec5WSM\nxUb7e3rOWZknL9vbpy2txz6yT5IODo6dPHJk5OPMx2Q/JxRKW2+8kfr1iRMjO48fH9l56NDY77JO\n7F66eCkSiayzt+/GSeBytyHhZ86c5gcvPiLf+au/msbGjc3L7fVVenqaVw0PL27btOk87LVevvjF\ni/OeNzpad826dZ/ZdOedn9k0PY3AT396dlfGBULv2F/iHY/HR+3nuLGWC9eLIZNY1B1YQvpsEnKW\nkzEcTu9Zvfriyc2bJy8V7Ntum57MftyFCwE8/vgSdHZO4Z57zqO5eeaTzOfs3TvxyYMPNl32QTA1\nFZiMRJZty251SNiXAHOaJiWnExZ18rSWlpYzt9xycXUkMv3Uj3509tw995xHX9+SxoGBiQt2wd69\ne+L3q1dfPFlfD7z8cj1uvHEmcfPNMzfffPPFXUNDZ+Mvvnh21zXXzDw3MlL6f/dCqz2a4sY2aAHx\nal/IKwNC+mwSclaSMXuhr/b2aSsWG4nZFxFlXUg0rx9dSt86O6fTa5sY5WxDws+cOc0PXnxEvjR7\n1B14PRJZdmmeeqH1ViqZoePGWi5cL4ZMCcx9QrlueHjYWrNmTaAmGyfPcZrSl2P9lkQ1V0Ak8iqn\n2skjdaq5Yr4kOvNIu77eakynZ2eNhMOpgkfcnP9NC45X+0JeGRDSZ5OQM1/GUr8Yo9jeOOd/e2Mw\np9nBeerkK6XM6eb8b1qIWNQdWELmrkrImS+j16b0SdiXAHOaJiWnExZ1qrlSvgKulA+A7Mc++eTE\nyWDQaozHx/oLfd0dkWRlzX5RSq0GsAvAr7TW38y4/1YA2+dubtdav5bvNaTMfpGyxrKEnKYylnLy\n035sfb3V+N57dZ09Pc2rgMKzZyTsS4A5TZOS06l2lnuk3gDg8cw7lFJ1APoA3DY3HlFKeb5okzyz\nc7pn12lxms1iP3ZqKjDZ09O8iv118ruyirrW+hiAZNbdKwC8o7We1FpPAngXwPUV5qs5CZ/cgIyc\nEjIC1cuZSqXa4vGxflPtn4W+P02TktNJwXnqSqkvA/hW1t2btNa/yfHwVgAjSqmBudujAK4CcKri\nlEQVKvaLPKqlmLn4RCYULOpa66MAjhb5Wp8AuBLAAwACAJ4CUPA/bGYPKxAIdACfflp65bZ9n1fy\nFLjdA+D/eChPrts3Wpa1uxbbb21tvSESifzD0NAr/xkAtmy541//5m/iN+R6fPbP3sT2f/KTxPcf\nfvjmy77H9bHHjn//61+P3C1xf5Zyuxr702//P0u9XUjZywQopToA3GGfKFVKBQH8C4BbMVvUj2qt\nv5Tv+TxRapaEnBIyAtXJGY+P9UejS7faRT0UmsHQ0PijmWvXlGoh789qkJKzKidKlVKbATwC4CtK\nqacBQGudxuyJ0qMAjsz9vXgSfsiAjJylZDTdfy5FNfZlNebiS/iZA8zpNi7oRZ7j18W7uA4NmVCt\nKY0LRmY/0Msk5Cw2Y60v76/WvixlKmYxJPzMAeZ0G4s6EZGPsKg7kNJnk5Cz2Iy1XgtGwr4EmNM0\nKTmdcD118pxKvqWIaKHjkboDKX02CTlLyWi6/1wKCfsSYE7TpOR0wqJOROQjLOoOpPTZJOSUkBFg\nTtOY010s6kREPsKi7kBKn01CTgkZAeY0jTndxaJOROQjLOoOpPTZJOSUkBFgTtOY010s6kREPsKi\n7kBKn01CTgkZAeY0jTndxaJOROQjLOoOpPTZJOSUkBFgTtOY010s6kREPsKi7kBKn01CTgkZAeY0\njTndxaJOROQjLOoOpPTZJOSUkBFgTtOY010s6kREPsKi7kBKn01CTgkZAeY0jTndxaJOROQjLOoO\npPTZJOSUkBFgTtOY010s6kREPsKi7kBKn01CTgkZAeY0jTndxaJOROQjLOoOpPTZJOSUkBFgTtOY\n010s6kREPsKi7kBKn01CTgkZAeY0jTndxaJOROQjLOoOpPTZJOSUkBFgTtOY010s6kREPsKi7kBK\nn01CTgkZAeY0jTndxaJOROQjLOoOpPTZJOSUkBFgTtOY010s6kREPrKo3CcqpVYD2AXgV1rrb2bc\n/xyAVQDOA3hOa/1PlYaspUAg0CHhE1xCTgkZAeY0jTndVXZRB9AA4HEAN2fdbwH4mtb6gwpem4iI\nylB2+0VrfQxAMs9fB8p9Xa+R8sktIaeEjABzmsac7nI8UldKfRnAt7Lu3qS1/k2ep5wF8IJSKgmg\nV2v9hwozEhFRkRyLutb6KICjxb6g1nojACilbgSwA8D6stN5gJQ+m4ScEjICzGkac7orYFlW2U9W\nSnUAuCPzRGnG37UD+I7WWuV67vDwcPkbJiJawNasWZO3xV12UVdKbQZwO4AQZmfA3D93/yCAP8Fs\nG2aD1vr9sjZAREQlq+hInYiIvIUXHxER+QiLOhGRj7CoExH5SCVXlJYl3/ICc3/XAOAdAN/TWj/p\ndrasLPmWQfgBZpdBqAPwda31v9UoYqGMtwLYPndzu9b6tVrky0UpdTeADQAuAtiqtf5ljSPlpJS6\nGsABzL4r5x6/AAACr0lEQVRHfq21frDGkfLy0vsmHy+9b3Lx8nvGVuw+rMWRur28QC5/B+B/Y3ap\ngVrLmVNr/Xda61sA9AGYN5XTZfMyKqXqMJvttrnxiFLKS1f4/j1ml5a4HcBjNc5SyE4A/6C1Xu3l\ngj7HS++bnDz2vrmMgPcMgOL3oetFPd/yAkqpJgBfBvBTeGCZAYdlEIDZKZtTLsXJKU/GFQDe0VpP\naq0nAbwL4HrXw+X3WwB/CaATwFs1zpKTUioI4PNa6+O1zuLEa++bItT8fZOD198z2Qruw6q1X8pY\nXmAjgH8E8B+rlSmXMnLa7gWwpzqpLldixlYAI0qpgbnbowCuAnCqihHnyZcZwBEAPQDqAdS8VZAn\nZz+AJUqpnwBYBuB/aq1fcj1chgL783bU4H2TTxH/V11735TAE++ZEhTch1Ur6qUsL6CUugJARGv9\nXaXUf6tWplxKXQYBAJRSXwHwe631yeqkulyJGT8BcCWABzB75PYUgDNVipZXrsxKqesAdGqt/3ru\n9r8opY7NHR3VRJ6cizD7xv4qgCCAN5RSr3ow5xUAVmutn3D7fZNPof+rbr9vSuCJ90wxitmHtZr9\nkv1r4pcwe2R0ELP9wa8rpb7gfqx55v06q5T6TwD+Umu9uwZ5csnO+C6AlRm3V3hoUbUg5g4k5nqW\njfBgH1hrfRHA/wMQ0lpPAbhQ40j5ePV9M48H3zeZvPyeuaTYfej6FaX5lhfI+Pt7ADRrrZ9yNViW\nAssg/Btm3/AzAP6vvYCZxzLeBuDbcw/rmzt68gSl1MMAIpg9oBjUWj9X20S5KaU+C+AHAK4AoLXW\nXmsZXMYr75t8vPS+ycXL7xlbsfuQywQQEfkILz4iIvIRFnUiIh9hUSci8hEWdSIiH2FRJyLyERZ1\nIiIfYVEnIvIRFnUiIh/5/wJaTmLmsj2EAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x112ab0910>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data = np.array(test_rows.collect())\n",
    "plt.scatter(data[:, 0], data[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x112df6ad0>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAEBCAYAAACHTjUfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X1wU+edL/CvJGxjm5LIMfeqM9naSQN4qsxOZu7OLklF\n64wpbCbusqSrU+/MbXLbyU73hgHboS3QhRDXaYFCal42mWabkGzIBPfxJm25aS9vTmmkQJjuztzJ\njbY0NBvI7UzYhOjYxi9gWz73D+sQRZZ09PLo6DyH72fmmUFC0vlyjH46/p3nPPIYhgEiInIHb6UD\nEBGRPCzqREQuwqJOROQiLOpERC7Cok5E5CIs6kRELjKv2CdqmrYcwOMAfiOE+HbyvhUAtiUfsk0I\n8WrpEYmIKF+lHKnXANhu3tA0zQugB8DK5HhU0zRPafGIiKgQRRd1IcQJAPGUuxYDeFsIMSGEmADw\nDoDbSsxHREQFKLr9kkEDgCFN0/qSt4cB3ATgnMRtEBFRDjKL+kcAbgTwEAAPgCcBXJL4+kREZKHU\nop7aM38HwJKU24uFEH/I9sTBwUEuOkNEVIS2tras5ytLmf2yEcA9AAKapi0UQnxT07QeAMeTD3m0\nlGBERDSX1QGxp1KrNA4ODhoqFHWPx9NqGMbJSuewokJOFTICzCkbc8plVTt58RERkYuwqFtQ4ZMb\nUCOnChkB5pSNOe3Fok5E5CIs6hY8Hk9rpTPkQ4WcKmQEmFM25rQXizoRkYuwqFtQpc+mQk4VMgLM\nKRtz2otFnYjIRVjULajSZ1MhpwoZAeaUjTntxaJOROQiLOoWVOmzqZBThYwAc8rGnPZiUScichEW\ndQuq9NlUyKlCRoA5ZWNOe7GoExG5CIu6BVX6bCrkVCEjwJyyMae9WNSJiFyERd2CKn02FXKqkBFg\nTtmY014s6kRELsKibkGVPpsKOVXICDCnbMxpLxZ1IiIXYVG3oEqfTYWcKmQEmFM25rQXizoRkYuw\nqFtQpc+mQk4VMgLMKRtz2otFnYjIRVjULajSZ1MhpwoZAeaUjTntxaJOROQi8yodwOlU6bOpkFOF\njABzypZPTl3XG2MxXycABIOJvX6//1LZg6VRZX9aYVEnoorSdb3x6NGqk93ddUEA6OsbX7Nqld5a\nicLuBmy/WFClz6ZCThUyAswpm1XOWMzX2d1dF7x40YuLF73o7q4LmkftdlJlf1rhkToRFcwJ7RLK\njEfqFlTps6mQU4WMAHNaMdsl4fCCLeHwgi1Hj1ad1HW9MdvjrXIGg4m9fX3jsUBgBoHADPr6xmPB\nYGKv9OAWVPm5W+GROhEVJLVdAgDd3XXBgYHRzlAIW4t5Pb/ff2nVKr11YGBU2pH/9fybBI/ULajS\nZ1MhpwoZAeaULZ+cfr//Uii0cGsotHCrjIJeyG8SheRUAY/UiaggwWBi7xNPjP1NJDKvBQCWL58+\nW4l2STayf5NQDYu6BVX6bCrkVCEjwJymXC2MqSl4+vtrAADLlk17KplTFlVyWmH7hYjmyNXCiMV8\nnV1d9UvNKYhdXfVLKzEFMRunnHitFBZ1C6r02VTIqUJGgDkBuXPHy5lT1/XGaHSkNxod6TU/dGZP\nvE61DgyMPjYwMPrYqlVTeV3IpMrP3QrbL0RUkOSR8JqUK0ClHQkXMmsl15Wosyder48e+hyGYUgd\n4XD4uXA4fDocDv86HA4/kO1xJ06cMGRvm4ODQ86Ix+ONhw5dfisQSBiBQMI4dOjyW/F4vDH17yOR\n4d5IZLg39f5ybjN9RCLDvYFAwgAMAzCMQCBhRCLDvcVsV/a/pZzDqnaW40jdAPBVIcR7ZXhtIrKB\n1dzxQo+EMx2Bm/f5fEatzwdMTnr/wu5ZK25cd6Zc7ZecZ8NV4vF4Wg0FzoqrkFOFjABzmmS0MGaL\n5ttP+Hyfu3v9+vpFwLXCed/Ro1Uvm8V048YJXLxYWNlIbwPt2zf2YTCYOFjIa7hx+mM5TpReBvCi\npmn/S9O028rw+kSkAPMo+MyZP9XWr69f9MmTrt5nUk/E7txZi4kJDzZunEC+s1aSJ0TvO3Bg9MOO\njqt45JHaRUePVr2cz4VGrlauvk84HL4jHA7/LFdfCECreRtAK2/zNm8753YoFFpt9ppDodDqQp9/\n4EDk+UAgYTz44ISR3vv+5S/ffTP9vgcfnDBaWqaMSGQocuBA5PlQKLTaanvZ+ur5/nuz9fGdsP+z\n3bbqqWf9i1JHOBxuCYfDItvf80QpB4dzR6EnLTMNs+A2NCSMXbvGjLTXWpL6+n19o0ZLy1TB25Fx\nstRtJ0qlbzAcDveHw+HfhMPhV8LhcFOxwZwyUj8tnTxUyKlCRuacHbKKpVm4W1qmjF/9aviDU6eG\ndpuF0yymp04N7T5zZmh3MUVVxoePaj9322e/CCE6ZL8mETlTrnnl5gyaH/zg1I8WL/7TC8XMoLGa\nt16OFR6V59RPGw4OjsqNfI6AZR4lF5vhehyVmKdORGWi63qdLxbb7D1/fjkAzDQ3RxLB4Ha/3z8u\nczv5HAGXazrgx0fn3taenvnByUngvvsmceaML3jrrYlNf/7n+FYpr+92LOoWOGdZHhUyAs7Nqet6\n3bxodLBu3bplvgsXAACJpqYvju/f36aHQitkFnYZXzJhvsa5c282/fVfBx/O5zV0XV/yxhu+6LFj\nVYuOHKnC+vVXcOWKBz/8YS0A4K67pu9fvFjfUY4Wi1N/7gVz6q8QThlQ5OSJCjlVyOjknMORSO90\nU5Nx7exlckw3NRnDkcj3ZG0n37bH7AyWkSsfP27kSjweX1LIa6RvV4iRD8zn7No1ZmzcODZnOmQx\nSwGo/HNPH1a1k6s0WjAU+eRWIacKGQHn5vSeP7/cPEJP5btwAd7z578gazv5rtAYi/m+1tNTW9Pe\nPon29kn09NTWxGK+r+X7GukrLMZivs7Ui5Qef3w+PvvZGVn/LEtO/bkXiu0XIira0JAXTz89HwAQ\nCORfgDOtuXLrrYkj6Y9rbk5c2rNn7KOurvqlycfNucpU1veRuuZ7TZ36K4RTBhT5lUyFnCpkdHJO\nB7ZfGvv7R86aj+vvHzmbOgc99TVefnn43OuvD52ORIYi8Xh8SaZ58KdODe1OfY4QIx/E4/EluS4O\nkjVDJh6PNz777PvvqjDThrNfiFwiEQxuH9+/v61u3bo7U06UYnz//tOJYHCHrO0UMve7qgpGR8fV\na3/O9Br/+Z/nbp+cXHLPV76y4DYA2Lt37N9vuWXm+Tn/voRnIvnlFnO2m21GjawZOLGYr3Pz5v/S\n7IaFvVjULRiK9NlUyKlCRsC5Of1+/7geCq0Ye+GFTWYPfaa5+bVEMLijHFMarQpaLObrXLu2vsUs\nhEeOTLf85CdjPwNGTpoFORTC1mj0tkg4XF9jPq6zs9538ODlr+7ZM/b79LaKrC+3cE0rpRhO/RWC\ng4PD2SO1hdLQMLt+S6b2RSQyFElvtXR1jRvm0gClrLmSpf2ypJiZN6pc6FSxBb1KDeaUAYf2V1XM\nqUJG5sx/pBbCrq7xrFMPv/GNb/z3n/50ZDp1quLsaoxypiam99yLXbcmdVVKpxZ0w2BPnYjK5JO9\nd6O1v78mlOlxBw4c+OPu3bs//+yzo68ePVpV98wz1di27Yq07zWd27IZKep1otHocCi0sKS1aBzB\nqZ82HBwc6ox814qx40i4XK0Up7RorGqnxzAM68pfBoODg0ZbW5trvvaO6HrnpKPYcmSJRkd6w+EF\nW8wTvoHADAYGRh+zOrqXzap2sv1iQZX1IFTIqUJGgDmLlW3mSiVyFjOLxmn7s1hcJoCIKA/JL7qO\n5fsdqpXCI3ULqnxyq5BThYwAc8rmlpyqfCEHizoRXVdK6bfLujiqnNh+seDxeFornSEfKuRUISPA\nnLI5Kae5kFg4vGBLOLxgy9GjVSd1XW8EnJWzFCzqRKSs9OV7rR6f77LCKmP7xYJb+oFOoEJGgDll\nK1fOTMv3rlqltxbb51Zlf1rhkToRKamYo25VZrCUgkXdgip9NhVyqpARYE7ZnJRzdgbLVOvAwOhj\nAwOjj61aNXXtyN5JOUvB9gsRKSl51L0mpf2S11G3CjNYSsGibkGVPpsKOVXICDCnbOXKKXveuCr7\n0wqLOhEpy+1H3cVgT92CKn02FXKqkBFgTtmY014s6kRELsKibkGVPpsKOVXICDCnbMxpLxZ1IiIX\nYVG3oEqfTYWcKmQEmFM25rQXizoRkYuwqFtQpc+mQk4VMgLMKRtz2otFnYjIRVjULajSZ1MhpwoZ\nAeaUjTntxaJOROQi0pcJ0DRtBYBtyZvbhBCvyt6GnVTps6mQU4WMAHPKxpz2knqkrmmaF0APgJXJ\n8aimaR6Z2yAiouxkt18WA3hbCDEhhJgA8A6A2yRvw1aq9NlUyKlCRoA5ZWNOe8luvzQAGNI0rS95\nexjATQDOSd4OERFlILuofwTgRgAPAfAAeBJA0esbO4EqfTYVcqqQEWBO2ZjTXrLbL+8AWJJye7EQ\n4g/ZHpz6647H42nlbd7mbd7mbevbuXgMw8jncXnTNG0lgEeSN3uEEMczPW5wcNBoa2tz/ElUj8fT\nqsInuAo5VcgIMKdszCmXVe2UPqVRCHEMwDHZr0tERNZ48ZEFFT65ATVyqpARYE7ZmNNeLOpERC7C\nom4h35MTlaZCThUyAswpG3Pai0WdiMhFWNQtqNJnUyGnChkB5pSNOe3Fok5E5CIs6hZU6bOpkFOF\njABzysac9mJRJyJyERZ1C6r02VTIqUJGgDllY057sagTEbkIi7oFVfpsKuRUISPAnLIxp71Y1ImI\nXET6gl5uo0qfrdw5dV1vjMV8nQAQDCb2+v3+gtfJ576UiznlUiWnFRZ1sqTreuPRo1Unu7vrggDQ\n1ze+ZtUqvbWYwk5E5cX2iwVV+mzlzBmL+Tq7u+uCFy96cfGiF93ddUHzqL0Q3JdyMadcquS0wqJO\nROQiLOoWVOmzlTNnMJjY29c3HgsEZhAIzKCvbzwWDCb2Fvo63JdyMadcquS0wp46WfL7/ZdWrdJb\nBwZGSzpRSkTlx6JuwePxtMbj8bdKnflRqEJnm5T7+xX9fv+lUAhbS3kNVb4DkjnlYk57sahbCIVC\nN9g986OY2SahUOiGaHSkF+CRNNF1zTCMiowTJ04Yldp2ISMSGe4NBBIGYBiAYQQCCSMSGe510jbj\n8XjjoUOX3woEEkYgkDAOHbr8Vjweb6z0vuPg4JA/rGonT5SWia7rjdHoSG80OtKr63pjObdlTjmc\nnATa2ydx5owveO6cd1M5t0lEzsSibmHz5nv/tdCZH2b7JBxesCUcXrDl6NGqk4UU9vTZJk88MXbW\n5zNqc31A3HjjDDZvvoJXXqlGf38N3nvP+z91XV9S6L+3nFSZB8yccjGnzZz6K4RTBoDWeDzeGIkM\n90Yiw735tDWKbdmkbWdJJDLce+bM0O7+/pGzuVor8Xi88fDhS3r6Nn/1q+EPnNSGAdBa6QzMyZyq\n57SqnTxRasFIng0vdeaHlcwnR6daYzFfZ1dX/dKLF2d/qerurgsODIx2pubx+/2XGhuHnwGwIfU1\njx2rWnTTTcamycmRCaDyJ1ANRWYWMKdczGkvtl/KIBhMHOzrG7v6cctm7GowmDiY6zmlXorf0jKz\nY9++sQ/NbW7YcAWnT/tw4YL3/mLbQESkHhZ1C4X02cyTo7GY95mentqa9vZJtLdPoqentiYW830t\n13Orq43ajo6rePDBK2homLl2f75XczY0NNy+YsX0l1944fL49743jn/5l3nYtu3Kh+vX1y8q9oOi\nkJO9+TxWlZ4lc8rFnPZi+0WS1PZJR8dVDA158fTT8wEAgcCM5XPffXdee39/DQBg69YJfOYzM2fN\ndkk+V3Mm59P/Y3d3XR0A7Ns39mFDgzEA4KFS/z1A7rnyXMWRyEGc2uxXbaSeHG1oSBh9faOGeXKz\nv3/k7JkzQ7uznWjNdGL11Kmh3cVu33yN3/5WfyL1JGt//8jZfE+cZj7ZOxTJN3+55/JzcFyvgydK\ni6Drep0vFtvsPX9+OQDMNDdHEsHgdr/fP57P8+NxL556qgYDA5ejPh/OvP++997Vqz+1Acj/KDaR\n8EwkszTGYr7O6mqjNpGYvT/fE5667tGqq42hjo6rAICqKhj55M/mpZeqQ3/8Y+Ikj8KJHMypnzaV\nGvF4vG7k8OHT001NhnnoOd3UZIwcPnwqHo/X5Xhexqs68zmKzfZc8/6WlqlPHPlnmtYYCoVWCzHy\ngfmYXbvGjI0bx4xij6DTM+3aNWY0NCQKyp/+mlBkyhhzMqeTB68oLZAvFttct27dMt+FCx/fd+EC\n6tatu9MXi2W9SnO29z3VOjAw+tjAwOhjq1ZN5X00m+255oyYUGgaO3fWItcVo9FodLipaeb5jo6r\naG+fxPbt83H1qqfo/fBxpsvRjo6r2L59PuLxzP9dSvm3E5FkTv20qdS4fPDgyWuHtmnj8sGDJwt9\nvVLWZTGP8h98cMJoaZkydu0au3a0LsTInAuL0rf10ksjv7O6cKmc+Tk4OOQP9tQrrJS1yJPTGdf0\n9MwP9vRMoLOzHuZFSOvX1y/KdBFS+rYAoJR10LmWOpFaWNTTzDQ3RxJNTV9Mbb8AQKKpCTPNza9l\neo7V2ufFrkVuFtSbb57p9HqNFQCWZXusuRZ0pm2VejWsjLXUAXXWq2ZOuZjTXuypp0kEg9vH9+8/\nnWhq+vi+piaM799/OhEM7kh/fKmLd+Vr3jy8vmfP2O/Ni5D27Rv70Ooq1XzYuZokEZWfxzBKmuVW\ntMHBQaOtra34M3lllJzSuMl7/vwXAGCmufm1RDC4I9OUxmh0pDccXrDFbIsEAjMYGBh9LBRaWPKR\nbfpFPf/0T6PnqqrQcPRo1U1HjlTh+9+fOPvpT8/8spBpjrlev69vPMaTnETOZlU7pbdfNE17DsBS\nAFcAPCeE+GfZ2yg3v98/jlDoEYRCFc2Ruh4MALz6atXi/v4aXLzoRUPDDN57z9uydm19C1DcVZzp\nr9/dXRd8+eXLm+68E98qx7+HiMqvHO0XA8BXhRB3q1jQ01mtB5Hv2iyy3XffJHburEWx67oAmdeb\nGRry3F+uNowqa2swp1zMaa9ynSh1ZFulHMo5O8Sc/WK2R5Yvnz67bNm0p6urfumCBaW1zTKtN+Pz\nAQ8/XLfoJz8Z7yz3UsNEVB5FF3VN074E4Dtpd28AcBnAi5qmxQF0CyH+UEK+isvnbLis2SGZXjfb\nFMXqaqN22bLp9q6u+qUACv4NIX2d9p07axEOzy5EVi6qzCxgTrmY015FF3UhxHEAxzP81XoA0DTt\nDgC7AKwpdhuU+QPDvL14sb4j128IVlMt0/l8hX84EJHDlOuqp3A43BIOh0Wuq6KQstYCgFYn3jbv\nc0qeHLe7Um+HQqHVzz77/rvmlaDPPvv+u6FQaHW2v3/xxY/0Q4fe+Kl5tWiZ8nY5aH9lvZ3+s690\nHu5P7s/U21ZXlEqf0qhpWj+AT2O2DbNWCHEh0+OcPKUxlSoXJKTnzGeqZaFH8rIzOhVzysWcctk+\npVEI0SH7NSvJyT/k1KIcj8cLbpmU61xANk7el6mYUy7mtBevKFWU1ZWslZpqSUSVxaJuwalzV62+\nqNqJy+E6dV+mY065mNNeXNDLpezul7uBuc8OHIg06br+FvcZKSnXWdRyDqeup67KyLXOOddAl7s/\nOTicNPjNRy6Vq71i1ZqhubjPyC1Y1C04uc82O3tl4dZQaOHWhoaG2yudx4qT96WKVNmfzGkvFnUX\n4syXwnGfkVtwPXWX4onSwnGfkQpsv/iInMHuC4vcgPuM3IDtFwuq9NlUyKlCRoA5ZWNOe/FInVyL\n7RS6HrGoWzAUWQ9ChZx2Zszw/at5f92fCvsSYE7ZVMlphe0XciWreee6rjdGoyO90ehIr+yv7yvn\naxNZYVG3oEqfTYWcTslotRhaKTmtXlsmp+xPK8xpLxZ1qphsR7QyjnRzzTsv59WjvDKVKo09dQuq\n9NlUyJmaMVvPGwCK7YWnKvQLwX0+ozZTTidjTrlUyWnJqYvScLh7RCLDvYFAwgAMAzCMQCBhRCLD\nvdnuz/Qa8Xi80XxOPB5fkvLnnAtxxePxxv7+kbPm4l19faPGSy+N/E7GAl5cGIyj3IMLepVIlT6b\nCjllZpzbu5735t/9XV1efWy/33/plltmXunouIr29kn09tZi7dr6FrNNUkpOO9exV+FnDjCn3dh+\noYpI9rzXpLRZrvW8s92fKrV3DQDd3fU17e2TePrp+ejurgsODIx2pl4dmj5nfXLSN9HfX4PU73A1\nhUKhG6LRkV7zscW0fnhlKlUKi7oFQ5E+W6Vz5nOhT2rGXD3vQnrh+Wab26efui/Th4eu641r1/7v\n74fDpfX07VDpn3m+mNNmTu0LcagzKtFHnrvNkSstLVMZvzAkEhk6nalPn7knPxTJt6fPwVGJYVU7\neaRuwePxtBoKfIJXMufcVsjc9ofsjBmO9A/efPP415J/3uv3+y/put547Ni86BtvzFua/nyfz6hN\n+c3i4NGjVS93d9cFOzquyohnC/7flEuVnFZY1ElZ6b3rYFDfG4v5OmMxX6f5566u+qWTk8DmzVfw\n+OPzAQD79o199P773nvXrq1vAYADB0a/2d1dt+jiRS+ef74GW7dOYOfO2RmOXFedVMOibkGVT+5K\n5sx10jNVsRl1XW88d867aXISf+Hz4UxLy8yO9B53et98z56x8C23JC53dFzF6KgHP/5xNTo6ruJz\nn0vgT/4kcW7NmoXLzN8sjh2rWmS+TjzuxVNP1WBg4HIU8Jx08kJg/L8plyo5rbCoU8kKvdCnELqu\nN/761/Mi773nbUkePYf27BlrX7lS/6tYzHet3XL2rHdTagvo0Udrl27cOIH+/hoAwMaNE6ipMXDT\nTcZZr9fzOoBl5jaOHKnCvn1jH65fX78IALZtuxILBmfWmP8GO1Z75IqSJI1Tm/1OGQBaK53BLTmL\nyRiJDPd2dY0bqScvW1qmDCFGPjBPkvb3j5w9dmzow9THpD8nEEgYr7+u//bMmaHdp04N7X7ppZHf\npZ3YvXbxUigUWm1u346TwMVuQ4WfOXPKH7z4iFznL/9yCuvX1y8y11fp6qpfOjhY1bhhwxWYa73c\neef0nOcND3ubVq/+1Ib77vvUhqkpeH7xi8uPp1wg9Lb5Jd7RaHTYfI4da7lwvRiSiUXdgqFIn02F\nnMVkDAYTe5cvnz67cePEtYK9cuXURPrjrl71YPv2+Whvn8QDD1xBff3MR6nP2bdv7KOHH677xAfB\n5KRnIhRauDW91aHCvgSYUzZVclphUSdH8/v9l+6+e3p5KDT15AsvXB5/4IEr6OmZX9vXN3bVLNh7\n9oz9fvny6bPV1cArr1TjjjtmYnfdNXPXXXdNPz4wcDn68suXH29qmnluaKjw/+65VnuUxY5t0HXE\nqX0hpwwo0mdTIWcpGdMX+mppmTIikaGIeRFR2oVEc/rRhfSt03NavbaMUcw2VPiZM6f8wYuPyJVm\nj7o9J0Ohhdfmqedab6WUGTp2rOXC9WJIFk/yE8p2g4ODRltbm6ciGyfHsZrSl2H9llg5V0Akciqr\n2skjdaq4fL4kOvVIu7raqE0kZmeNBIN6ziNuzv+m645T+0JOGVCkz6ZCzmwZC/1ijHx745z/7YzB\nnHIH56mTqxQyp5vzv+l6xKJuwVBk7qoKObNldNqUPhX2JcCcsqmS0wqLOlVcIV8BV8gHQPpjn3hi\n7KzPZ9RGoyO9ub7ujkhlRc1+0TRtOYDHAfxGCPHtlPtXANiWvLlNCPFqttdQZfaLKmssq5BTVsZC\nTn6aj62uNmrffdfb3tVVvxTIPXtGhX0JMKdsquS0qp3FHqnXANieeoemaV4APQBWJsejmqY5vmiT\nembndM+u02I1m8V87OSkZ6Krq34p++vkdkUVdSHECQDxtLsXA3hbCDEhhJgA8A6A20rMV3EqfHID\nauRUISNQvpy6rjdGoyO9sto/1/v+lE2VnFZyzlPXNO1LAL6TdvcGIcSbGR7eAGBI07S+5O1hADcB\nOFdySqIS5ftFHuWSz1x8IhlyFnUhxHEAx/N8rY8A3AjgIQAeAE8CyPkfNrWH5fF4WoGPPy2dctu8\nzyl5ctzuAvB/HJQn0+07DMPYU4ntNzQ03B4Khf5hYOCXfwYAmzff+69/+7fR2zM9Pv1nL2P7P/95\n7Eff/e5dn/ge1x/84NSPvv710P0q7s9Cbpdjf7rt/2eht3MpepkATdNaAdxrnijVNM0H4DUAKzBb\n1I8LIT6f7fk8USqXCjlVyAiUJ2c0OtIbDi/YYhb1QGAGAwOjj6WuXVOo63l/loMqOctyolTTtI0A\nHgXwZU3TngIAIUQCsydKjwM4lvx75anwQwbUyFlIRtn950KUY1+WYy6+Cj9zgDntxgW9yHHcungX\n16EhGco1pfG6kdoPdDIVcuabsdKX95drXxYyFTMfKvzMAea0G1dpJFfTdb3OF4tt9p4/vxwAZpqb\nI4lgcLvf7x+vdDaissi12lc5hyqrNHLYP4pdXTHD69SNHD58erqpyTCXgJxuajJGDh8+FY/H6yr9\n7+TgKGZwlUZSTiFrweTii8U2161bt8x34cLH9124gLp16+70xWKbpIYmcggWdQuq9NlUyFlIRhn9\nZ+/588tTC7rJd+ECvOfPf0FGzkpiTrlUyWmFRZ2IyEVY1C0YisxdVSGn3RlnmpsjiaamOfcnmpow\n09z8WrbnqbAvAeaUTZWcVljUybUSweD28f37T6cW9kRTE8b37z+dCAZ3VDAaUdlwSqMFVS4dViGn\n3Rn9fv+4HgqtGHvhhU1mD32mufm1RDC4I9eURhX2JcCcsqmS0wqLOrma3+8fRyj0CEKhSkchsgXb\nLxZU+eRWIacKGQHmlI057cWiTkTkIizqFlSZu6pCThUyAswpG3Pai0WdiMhFWNQtqNJnUyGnChkB\n5pSNOe3Fok5E5CIs6hZU6bOpkFOFjABzysac9mJRJyJyERZ1C6r02VTIqUJGgDllY057sagTEbkI\ni7oFVfqbq5AoAAAE00lEQVRsKuRUISPAnLIxp71Y1ImIXIRF3YIqfTYVcqqQEWBO2ZjTXizqREQu\nwqJuQZU+mwo5VcgIMKdszGkvFnUiIhdhUbegSp9NhZwqZASYUzbmtBeLOhGRi7CoW1Clz6ZCThUy\nAswpG3Pai0WdiMhFWNQtqNJnUyGnChkB5pSNOe3Fok5E5CIs6hZU6bOpkFOFjABzysac9mJRJyJy\nERZ1C6r02VTIqUJGgDllY057sagTEbnIvGKfqGnacgCPA/iNEOLbKfc/B2ApgCsAnhNC/HOpISvJ\n4/G0qvAJrkJOFTICzCkbc9qr6KIOoAbAdgB3pd1vAPiqEOK9El6biIiKUHT7RQhxAkA8y197in1d\np1Hlk1uFnCpkBJhTNua0l+WRuqZpXwLwnbS7Nwgh3szylMsAXtQ0LQ6gWwjxhxIzEhFRniyLuhDi\nOIDj+b6gEGI9AGiadgeAXQDWFJ3OAVTps6mQU4WMAHPKxpz28hiGUfSTNU1rBXBv6onSlL9rAfA9\nIYSW6bmDg4PFb5iI6DrW1taWtcVddFHXNG0jgHsABDA7A+abyfv7AXwas22YtUKIC0VtgIiIClbS\nkToRETkLLz4iInIRFnUiIhdhUScicpFSrigtSrblBZJ/VwPgbQA/FEI8YXe2tCzZlkH4MWaXQfAC\n+LoQ4j8qFDFXxhUAtiVvbhNCvFqJfJlomnY/gLUApgFsEUL8usKRMtI07WYABzH7HvmtEOLhCkfK\nyknvm2yc9L7JxMnvGVO++7ASR+rm8gKZ/D2Af8PsUgOVljGnEOLvhRB3A+gBMGcqp83mZNQ0zYvZ\nbCuT41FN05x0he+3MLu0xD0AflDhLLnsBvAPQojlTi7oSU5632TksPfNJyjwngGQ/z60vahnW15A\n07Q6AF8C8As4YJkBi2UQgNkpm5M2xckoS8bFAN4WQkwIISYAvAPgNtvDZffvAL4IoB3AGxXOkpGm\naT4AnxVCnKp0FitOe9/koeLvmwyc/p5Jl3Mflq39UsTyAusB/COA/1quTJkUkdP0DQB7y5PqkwrM\n2ABgSNO0vuTtYQA3AThXxohzZMsM4BiALgDVACreKsiSsxfAfE3Tfg5gIYD9Qoif2R4uRY79eQ8q\n8L7JJo//q7a9bwrgiPdMAXLuw7IV9UKWF9A07QYAISHEDk3T/ke5MmVS6DIIAKBp2pcB/F4IcbY8\nqT6pwIwfAbgRwEOYPXJ7EsClMkXLKlNmTdNuBdAuhPir5O3XNE07kTw6qogsOedh9o39FQA+AK9r\nmnbEgTlvALBcCLHT7vdNNrn+r9r9vimAI94z+chnH1Zq9kv6r4mfx+yR0SHM9ge/rmna5+yPNcec\nX2c1TftvAL4ohNhTgTyZpGd8B8CSlNuLHbSomg/JA4lkz7IWDuwDCyGmAfw/AAEhxCSAqxWOlI1T\n3zdzOPB9k8rJ75lr8t2Htl9Rmm15gZS/fwBAvRDiSVuDpcmxDMJ/YPYNPwPg/5oLmDks40oAjyQf\n1pM8enIETdO+CyCE2QOKfiHEc5VNlJmmaZ8B8GMANwAQQgintQw+wSnvm2yc9L7JxMnvGVO++5DL\nBBARuQgvPiIichEWdSIiF2FRJyJyERZ1IiIXYVEnInIRFnUiIhdhUScichEWdSIiF/n/R/E+73IQ\nuzsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x112ab0650>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "optimal_centroids = [np.array([-12, -4]), np.array([-6.5, -11]), np.array([-4, 4])]\n",
    "centroids = np.array(optimal_centroids)\n",
    "plt.scatter(data[:, 0], data[:,1])\n",
    "plt.scatter(centroids[:, 0], centroids[:, 1], c='r', s=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# intialize centroids\n",
    "\n",
    "def Kmeans(features, centers, num_iter):\n",
    "    error = []\n",
    "    \n",
    "    for i in xrange(num_iter):\n",
    "        def closest_centroid(point, centroids):\n",
    "            distances = [ np.sqrt(np.sum((point - c)**2)) for c in centroids ]\n",
    "            return (np.argmin(distances), np.min(distances))\n",
    "        \n",
    "        def compute_assignments(point, centroids):\n",
    "            closest = closest_centroid(point, centroids)\n",
    "            return (closest[0], (point, 1, closest[1]))\n",
    "        \n",
    "        # compute cluster assignments\n",
    "        assignments = features.map(lambda x: compute_assignments(x, centers))    \n",
    "            \n",
    "        # update centroids to mean of assigned points\n",
    "        means = assignments.reduceByKey(lambda x, y: (x[0] + y[0], x[1] + y[1], x[2] + y[2]))\n",
    "        centroids = means.map(lambda c: (c[0], c[1][0] / c[1][1])).collect()\n",
    "        \n",
    "        for i, p in centroids:\n",
    "            centers[i] = p\n",
    "        \n",
    "        WSSE = means.map(lambda x: x[1][2]).sum()\n",
    "        error.append(WSSE)\n",
    "        \n",
    "        if len(error) > 1 and error[-2] == error[-1]:\n",
    "            break\n",
    "            \n",
    "    return (assignments, centroids, error)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "res = Kmeans(test_rows, optimal_centroids, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(PythonRDD[21] at RDD at PythonRDD.scala:43,\n",
       " [(0, array([-10.04935243,  -3.85954095])),\n",
       "  (1, array([-6.58196786, -8.17239339])),\n",
       "  (2, array([-1.4710815 ,  4.33721882]))],\n",
       " [261.87984298397168, 108.56785435636178, 108.56785435636178])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x112f21810>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAEBCAYAAACHTjUfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X1wW+WdL/CvrMSO7UBi4/SqU4rDliSeirvLzO5MU1Bm\nzU2aLIO7NHD1NDt3gVsGll2yTeymbYBNGlxzSSEBJ3DD3S6QsIQh7uMNtBRanMRAK5Fspnvn7uXi\nxU2A4MCUlATJdvwSv8jn/mGdVJElHb08OjrPyfcz88xEsnz09XH00/HvPOeRxzAMEBGRO5SVOgAR\nEanDok5E5CIs6kRELsKiTkTkIizqREQuwqJOROQis/L9RiHEMgCPAviVlPJ78ftWANgSf8gWKeXr\nhUckIqJsFXKkXgFgq3lDCFEGoBXAyvh4QAjhKSweERHlIu+iLqU8BCCScNciAMeklKNSylEA7wO4\nqsB8RESUg7zbLynUAugXQrTHbw8AuAzAcYXPQUREGags6p8BmA/gHgAeAE8COKNw+0REZKHQop7Y\nM38fwOKE24uklO+l+8bu7m4uOkNElIfly5enPV9ZyOyXjQBuAOATQlwqpbxbCNEK4GD8IQ8UEoyI\niGayOiD2lGqVxu7ubkOHou7xeBoNw3iz1Dms6JBTh4wAc6rGnGpZ1U5efERE5CIs6hZ0eOcG9Mip\nQ0aAOVVjTnuxqBMRuQiLugWPx9NY6gzZ0CGnDhkB5lSNOe3Fok5E5CIs6hZ06bPpkFOHjABzqsac\n9mJRJyJyERZ1C7r02XTIqUNGgDlVY057sagTEbkIi7oFXfpsOuTUISPAnKoxp71Y1ImIXIRF3YIu\nfTYdcuqQEWBO1ZjTXizqREQuwqJuQZc+mw45dcgIMKdqzGkvFnUiIhdhUbegS59Nh5w6ZASYUzXm\ntBeLOhGRi7CoW9Clz6ZDTh0yAsypGnPai0WdiMhFWNQt6NJn0yGnDhkB5lSNOe3Fok5E5CIs6hZ0\n6bPpkFOHjABzqsac9mJRJyJyERZ1C7r02XTIqUNGgDlVY057sagTEbnIrFIHcDpd+mw65NQhI8Cc\nqmWTMxqN1vX0eNcDgN8f21lTU3Om6MGS6LI/rbCoE1FJRaPRuq6u2W+2tFT5AaC9fWT1qlXRxlIU\ndjdg+8WCLn02HXLqkBFgTtWscvb0eNe3tFT5T50qw6lTZWhpqfKbR+120mV/WuGROhHlzAntEkqN\nR+oWdOmz6ZBTh4wAc1ox2yXB4NxNweDcTV1ds9+MRqN16R5vldPvj+1sbx/p8fmm4PNNob19pMfv\nj+1UHtyCLr93KzxSJ6KcJLZLAKClpcrf2Tm0PhDA5ny2V1NTc2bVqmhjZ+eQsiP/i/kvCR6pW9Cl\nz6ZDTh0yAsypWjY5a2pqzgQCl24OBC7drKKg5/KXRC45dcAjdSLKid8f27lr1/B/DYVmNQDAsmWT\nvaVol6Sj+i8J3bCoW9Clz6ZDTh0yAsxpytTCmJiAp6OjAgCwdOmkp5Q5VdElpxW2X4hohkwtjJ4e\n7/rm5uol5hTE5ubqJaWYgpiOU068lgqLugVd+mw65NQhI8CcgNq548XMGY1G68LhwbZweLDNfNOZ\nPvE60djZOfRgZ+fQg6tWTWR1IZMuv3crbL8QUU7iR8KrE64AVXYknMuslUxXok6feL04eugzGIah\ndASDwWeDweCRYDD4RjAYvD3d4w4dOmSofm4ODg41IxKJ1O3bd/Ydny9m+HwxY9++s+9EIpG6xK+H\nQgNtodBAW+L9xXzO5BEKDbT5fDEDMAzAMHy+mBEKDbTl87yqf5ZiDqvaWYwjdQPAN6WUJ4uwbSKy\ngdXc8VyPhFMdgZv3eb1GpdcLjI+XfcXuWStuXHemWO2XjGfDdeLxeBoNDc6K65BTh4wAc5pUtDCm\ni+axXV7vl69ft656AXC+cN7c1TX7RbOYbtw4ilOncisbyW2gxx8fPu33x/bmsg03Tn8sxonSswBe\nEEL8XAhxVRG2T0QaMI+Cjx79Y7FuXfWCC0+6lj2TeCL24YcrMTrqwcaNo8h21kr8hOjNu3cPnV6z\nZgw/+EHlgq6u2S9mc6GRqxWr7xMMBq8JBoMvZeoLAWg0bwNo5G3e5m3n3A4EAjeZveZAIHBTrt+/\ne3foOZ8vZtx556iR3Pt+9dUTbyffd+edo0ZDw4QRCvWHdu8OPRcIBG6yer50ffVsf950fXwn7P90\nt6166mm/UOgIBoMNwWBQpvs6T5RycDh35HrSMtUwC25tbczYtm3YSNrW4sTtt7cPGQ0NEzk/j4qT\npW47UeoxDAMqCSE6AHwe022YtVLKvlSP6+7uNpYvX+743jv7q+rokBFgTgAIhwfbgsG5m8xes883\nhc7OoQcDgUtzOjlqnoScP38Kjz02cnr+fOO5hoapHyWeKJ09e+qSL8Q++sqck+99vmJq9GPjiive\niPn9W2tqakZyeQ5genpltvPSk+nye7eqncpPlEop16jeJhE5U6Z55eYMmoceOvzYokV/3Jfq635/\ndOuscLi76tvfXurt6wOA+lh9/XUjTzyxPBoIrABQlWneejFWeNSeU/+E4ODgKN3Ipv2iokUzEAq1\nTdbXG+f7J/ExWV9v9B8+vK3Q7btxWNVOLhNARDNkc6m9iqUEyj78cFn8CP0C3r4+jH105q7W1jn+\n8XGgqWkcR496/cePl91b4I/meizqFnRZD0KHnDpkBJgTUPMhE+a6LHv2hJ/LZ5rhu8cr5q1bdw6b\nNo3ilVfK0dFRgb6+stuKNWVRl9+7FRZ1IrpAth8y4ffH9ra3D4/9YV758Jh58U/iNu6//9pb021j\nauHCUKy+fkaGWH09pr70R+jrK8Mjj1TC/Gtg3brqBU5aEdKJWNQtGBqcDQf0yKlDRoA5s22r9PR4\nb21traxoahpHU9M4WlsrK3p6vLdmu41oNFrX4/3Ps6K7nvo4sbDH6uvxxtrnccd9V+JLX5oqxo+Y\nki6/dytcpZGI8tbfX4ann54DYHraY7YSpyLOmfNf8NP/1fXp5YPvjv17b9UXX/9oCdq3XIF58wws\nXBg7s2PH8GfNzdVLgNQrQqr6PFLXfK6pU8/gOmUg4aouJw8dcuqQkTmzn9USiUTqOjoGe83HdXQM\n9pqPS97Giy8OHH/rrf4joVB/KBKJLE510dDhw/3bE79HysFPI5HI4kwXB6mYgWNuZ8+eT07oMNOm\nFKs0EpHGcpn7PXs2jDVrxs7/O9U2fv/741ePjy++4ZZb5l4FADt3Dv/HlVdOPZe8rVjMMxqfcTPj\nedMtsKVqQa6eHu/6++773EI3LOzFom7B0KTPpkNOHTICzAlkt0JjT493/dq11Q1mIXzttcmGp54a\nfgkYfNMsyIEANofDV4WCweoK83Hr11d79+49+80dO4Z/m9xWUfXhFq5ppeSBJ0qJqGC1tVO4++4x\nBIOXBDLNmDG9+mp51ZVXTr2S60fOJUvzeaR7s5m9k8V29PxcU6f2hZwycJH3Vy+2jMyZ/UjsZzc3\nj8xYidFcWOuOO+7465/8ZHDS7Fdv2zYcX40x908pSpcjseee7yJfiatSOrWfbhjsqRNRkVzYezca\nOzoqAqket3v37o+3b99+3Z49Q693dc2ueuaZcmzZck7ZkfDMls1gXtsJh8MDVguWadHWceq7DQcH\nhz4j27Vi7DgSVjUjxq7t5jpsX3o3W7osvUtE2XHSUWwxsqhYjlgF25fedRtd1ljWIacOGQHmzFe6\nmSulyJnPLBqn7c98cfYLEVEWdJkhwyN1C7q8c+uQU4eMAHOq5pacunwgB4s6EV1UCum3q7o4qpjY\nfrGgyxrLOuTUISPAnKo5KWemZYWdlLMQLOpEpC3zgzjC4cG2bD48Q8WnNTkd2y8W3NIPdAIdMgLM\nqVqxciYu3wsA7e0jq1etiua13ACgz/60wiN1ItJSPkfdusxgKQSLugVd+mw65NQhI8CcqjkpZ6YP\n1HZSzkKw/UJEWoofda9OaL9kddStwwyWQrCoW9Clz6ZDTh0yAsypWrFyqp43rsv+tMKiTkTacvtR\ndz7YU7egS59Nh5w6ZASYUzXmtBeLOhGRi7CoW9Clz6ZDTh0yAsypGnPai0WdiMhFWNQt6NJn0yGn\nDhkB5lSNOe3Fok5E5CIs6hZ06bPpkFOHjABzqsac9mJRJyJyERZ1C7r02XTIqUNGgDlVY057sagT\nEbmI8mUChBArAGyJ39wipXxd9XPYSZc+mw45dcgIMKdqzGkvpUfqQogyAK0AVsbHA0IIj8rnICKi\n9FS3XxYBOCalHJVSjgJ4H8BVip/DVrr02XTIqUNGgDlVY057qW6/1ALoF0K0x28PALgMwHHFz0NE\nRCmoLuqfAZgP4B4AHgBPAsh7fWMn0KXPpkNOHTICzKkac9pLdfvlfQCLE24vklK+l+7BiX/ueDye\nRt7mbd7mbd62vp2JxzCMbB6XNSHESgA/iN9slVIeTPW47u5uY/ny5Y4/ierxeBp1eAfXIacOGQHm\nVI051bKqncqnNEopDwA4oHq7RERkjRcfWdDhnRvQI6cOGQHmVI057cWiTkTkIizqFrI9OVFqOuTU\nISPAnKoxp71Y1ImIXIRF3YIufTYdcuqQEWBO1ZjTXizqREQuwqJuQZc+mw45dcgIMKdqzGkvFnUi\nIhdhUbegS59Nh5w6ZASYUzXmtBeLOhGRi7CoW9Clz6ZDTh0yAsypGnPai0WdiMhFlC/o5Ta69NmK\nnTMajdb19HjXA4DfH9tZU1OT8zr53JdqMadauuS0wqJOlqLRaF1X1+w3W1qq/ADQ3j6yetWqaGM+\nhZ2IiovtFwu69NmKmbOnx7u+paXKf+pUGU6dKkNLS5XfPGrPBfelWsypli45rbCoExG5CIu6BV36\nbMXM6ffHdra3j/T4fFPw+abQ3j7S4/fHdua6He5LtZhTLV1yWmFPnSzV1NScWbUq2tjZOVTQiVIi\nKj4WdQsej6cxEom8U+jMj1zlOtuk2J+vWFNTcyYQwOZCtqHLZ0Ayp1rMaS8WdQuBQGCe3TM/8plt\nEggE5oXDg20Aj6SJLmqGYZRkHDp0yCjVc+cyQqGBNp8vZgCGARiGzxczQqGBNic9ZyQSqdu37+w7\nPl/M8Plixr59Z9+JRCJ1pd53HBwc6odV7eSJ0iKJRqN14fBgWzg82BaNRuuK+VzmlMPxcaCpaRxH\nj3r9x4+X3VvM5yQiZ2JRt3DffTf+W64zP8z2STA4d1MwOHdTV9fsN3Mp7MmzTXbtGu71eo3KTG8Q\n8+dP4b77zuGVV8rR0VGBkyfL/i4ajS7O9ectJl3mATOnWsxpM6f+CeGUAaAxEonUhUIDbaHQQFs2\nbY18WzZJz7M4FBpoO3q0f3tHx2BvptZKJBKpe/nlM9Hk5/zFLwY+dVIbBkBjqTMwJ3PqntOqdvJE\nqQUjfja80JkfVlKfHJ1o7Onxrm9url5y6tT0H1UtLVX+zs6h9Yl5ampqztTVDTwDYEPiNg8cmL3g\nssuMe8fHB0eB0p9ANTSZWcCcajGnvdh+KQK/P7a3vX147A8tm+Exvz+2N9P3FHopfkPD1I8ef3z4\ntPmcGzacw5EjXvT1ld2WbxuIiPTDom4hlz6beXK0p6fsmdbWyoqmpnE0NY2jtbWyoqfHe2um7y0v\nNyrXrBnDnXeeQ23t1Pn7s72as7a29uoVKya//vzzZ0d++MMR/Mu/zMKWLedOr1tXvSDfN4pcTvZm\n81hdepbMqRZz2ovtF0US2ydr1oyhv78MTz89BwDg801Zfu+JE7OaOjoqAACbN4/iiiumes12SfLV\nnABGBsPhtrIPP1wGAFMLF4a+8Y1vvN3VNXtLS0tVFQA8/vjw6dpaoxPAPYX+PEDmufJcxZHIQZza\n7NdtJJ4cra2NGe3tQ4Z5crOjY7D36NH+7elOtKY6sXr4cP/2VM8TiUSqBl9++chkfb1hfsNkfb1x\n+pUDHy1cOHnBNn7zm+iuxJOsHR2DvdmeOE19src/lG3+Ys/l5+C4WAdPlJZAJFKGH/+4Ap2dZ8Ne\nL45+8knZjTfddMkGIPuj2FjMMwr8YbmA8nKjMhYDvhD76Ctf+Pa3l3r7+s4/1tvXh5q1d13+N994\nE/fvXHj+/mjUI8rLjf41a8YAALNnwyjk59q/vzzw8cexN3kUTuRgTn23ccpAltOc0l3Vmc1RbLrv\nNe9vaJg4f+R/6IEu4/zGksbJ518dN7exbduwsXHjsJHvEXRypm3bho3a2lhO+fPdl6UezMmcTh48\nUrdJ+pUMB/P+3nB4sK2lpcrf1DSOhx+uxPg4cNll6Q+2q2aNn1qzZuyLQ0MebN06B7fdNqbg5zn7\n0v795YGtW+cgEilLeX6AqzgSOYhT323cMgpZl8U8yr/zzlGjoWHC2LZt2Nja/IGR2E9P7Kv3Hz68\nLfG59u8ffNfqwqVi5ufg4FA/rGqnxzAKarPmrbu721i+fLmnJE9us3w/tNmcVdLaOsff2jqK9eur\nMTDgwcutb+H6XX8Ns68eq6/HyBNPHJkMBFYAqEp8LmB6Dnyuz60iPxGpZ1U7WdQtZLPGcjGLnrnt\nsjJjxS23XLL01KkyVFYa2Pw3fbhr6f/pmzM5/OHUwoW/vua229764IMPulQ9bzHosl41c6rFnGpZ\n1U721Atk1xztWbPw1o4dwzXNzdVLAOCq6y477V3RuHJuTc0xADhx4kRjPtvlUTiRyzi1L6TLKOYc\n7eR+9ssvDxz75S8HzjQ3jxgNDRPG/v2D7x4+nH7+e67bZ7+cg8P5w/bZL0KIZwEsAXAOwLNSyn9W\n/RwXi8T1YADg9ddnL+roqMCpU2WorZ3CyZNlDWvXVjcA+f2FkLz9lpYq/4svnr33q1/Fd4vx8xBR\n8RVj7RcDwDellNe7oaBbrQeR7dosqt188/Q0x3zXdQFSrzfT3++5rViLfumytgZzqsWc9ipWT93x\nJ0BVKeYc7fgbxmqzX79s2WTv0qWTnubm6iVz5xZ2gjvVejNeL/Cd71QteOqpkfXFXmqYiIoj76Iu\nhPgagO8n3b0BwFkALwghIgBapJTvFZCv5IwszobX1NScKUYRTLOYFzo7h9aXlxuVS5dONpknTnP9\nCyF5nfaHH65EMDi9EFmxZLMvnYA51WJOe+Vd1KWUBwEcTPGldQAghLgGwDYAq/N9Dkr9hmHeXrQo\n+qNMfyHkOrPF6839zYGIHKZYZ2iDwWBDMBiUmc7gImGtBQCNTrxt3ueUPBluNyfeDgQCN+3Z88kJ\nc2bLnj2fnAgEAjel+/oLL3wW3bfvX39izn4pUt5mB+2vtLeTf/elzsP9yf2ZeNv2K0qFEB0APo/p\nNsxaKWVfqse56eIjJ0jOGQ4PtgWDczeZ7RWfbwqdnUMPBgKXnj/qt3uOuq770qmYUy1dctp+8ZGU\nco3qbZaSk3/JiUU5Eonk3DIp1rmAdJy8LxMxp1rMaS9+nJ2mzCtZ033+aKmmWhJRabGoW3Dq3FWr\nD6qenjkz0djZOfRgZ+fQg6tWTZT8gy2cui+TMadazGkvrv3iUlzTJXfmPtu9O1QfjUbf4T4jLWU6\ni1rM4Za1X0o1Mq3bwjVd1O5PDg4nDavayfaLpjK1V6xaMzQT9xm5BYu6BSf32aZnr1y6ORC4dHNt\nbe3Vpc5jxcn7Uke67E/mtBeLugtx5kvuuM/ILfjJRy7FE6W54z4jHfCTjy5Sdl9Y5AbcZ+QGbL9Y\n0KXPpkNOHTICzKkac9qLR+rkWmyn0MWIRd2Cocl6EDrktDNjIR8IrsO+BJhTNV1yWmH7hVzJat55\nNBqtC4cH28LhwTbVH99XzG0TWWFRt6BLn02HnE7JaLUYWiE5rbatklP2pxXmtBeLOpVMuiNaFUe6\nmeadF/PqUV6ZSqXGnroFXfpsOuRMzJiu5w0A+fbCE+X6geBer1GZKqeTMadauuS05NRFaTjcPUKh\ngTafL2YAhgEYhs8XM0KhgbZ096faRiQSqTO/JxKJLE74d8aFuCKRSF1Hx2CvuXhXe/uQsX//4Lsq\nFvDiwmAcxR5c0KtAuvTZdMipMuPM3vWst++6qyqrPnZNTc2ZK6+cemXNmjE0NY2jra0Sa9dWN5ht\nkkJy2rmOvQ6/c4A57cb2C5VEvOe9OqHNcr7nne7+RIm9awBoaamuaGoax9NPz0FLS5W/s3NofeLV\noclz1sfHvaMdHRVI/AxXUyAQmBcOD7aZj82n9cMrU6lUWNQtGJr02UqdM5sLfRIzZup559ILzzbb\nzD79xM2p3jyi0Wjd2rW//B/BYGE9fTuU+neeLea0mVP7Qhz6jFL0kWc+5+C5hoaJlB8YEgr1H0nV\np0/dk+8PZdvT5+AoxbCqnTxSt+DxeBoNDd7BS5lzZitkZvtDdcYUR/p7L7985Nb4v3cCGBk4cmT7\n0AeRv5s36K1av2YJfvjjhRgdnV7czus1KhP+stjb1TX7xZaWKv+aNWMq4tmC/zfV0iWnFRZ10lZy\n79rvj+7s6fGu7+0t23D1Z6EVVX//9382r68PXwTw5fp6XPvQ87h951fxyCMjn33ySdmNa9dWNwDA\n7t1Dd7e0VC04daoMzz1Xgc2bR/Hww9MzHLmuOmnHqX9CcOgzit1+iUQidUeP9m8PhfpDhw/3b0+1\n7cQMW5s/MCbr643zPZT4mKyvN9568Zhx5Ej0gnZMc/OIkXi7oWHCCIX6Q9lMj+TgsHuw/UJFl+uF\nPrmIRqN1b7wxK3TyZFlD/Og5sGPHcNPKldG/7Onxnm+39PaW3Wu2gP6k6j14+/pmbMvb14crRo+N\n/K7sc28BWGre/9prs/H448On162rXgAAW7ac6/H7p1abP4Mdqz1yRUlSxqnvNk4ZABpLncEtOfPJ\nGAoNtKU6kpZy8FPzL4OOjsHeAwf6T5uP+cX9B4zko3RznOl46eThw/3b9+8ffDfpL4vzFy8FAoGb\nzOe34yRwvs+hw++cOdUPXnxErvMXfzGBdeuqF5jrqzQ3Vy/p7p5dt2HDOfh8U/i/I1chVl8/4/ti\n9fX4pyPXfPHmmy/ZMDEBz89+dvbRhAuEjpkf4h0OhwfM77FjLReuF0MqsahbMDQ5G65Dznwy+v2x\nncuWTfZu3DgKc3GulSsnRpMfNzbmwdatc9DUNI6h6s/h9088O5lY2GP19Tj63b1o+6f6828E4+Oe\n0UDg0s3JrQ4d9iXAnKrpktMKizo5Wk1NzZnrr59cFghMPPn882dHbr/9HFpb51S2tw+PmUV+x47h\n3y5bNtlbXg688ko5rr7a6KkM/Mmf/u6Fnx/59CcvD3zW8dJHv3vh50f+26PXnp/SmK1Mqz2qYsdz\n0EXEqX0hpwxo0mfTIWchGZMX+kqeoZJ0IVHG2TFWfevknFbbVjHyeQ4dfufMqX5w9gu5Un9/GQDP\nm4HApefnqWdab6WQGTp2rOXC9WJIFU/8Hcp23d3dxvLly3P7W5hcy2pKX4r1W3qKuQIikVNZ1U4e\nqVPJZfMh0YlH2uXlRmUsNj1rxO+PZjzi5vxvuug4tS/klAFN+mw65EyXMdcPxsi2N875384YzKl2\ncJ46uUouc7o5/5suRizqFgxN5q7qkDNdRqdN6dNhXwLMqZouOa2wqFPJ5fIRcLm8ASQ/dteu4V6v\n16gMhwfbMn3cHZHO8pr9IoRYBuBRAL+SUn4v4f4VALbEb26RUr6ebhu6zH7RZY1lHXKqypjLyU/z\nseXlRuWJE2VNzc3VS4DMs2d02JcAc6qmS06r2pnvkXoFgK2JdwghygC0AlgZHw8IIRxftEk/03O6\np9dpsZrNYj52fNwz2txcvYT9dXK7vIq6lPIQgEjS3YsAHJNSjkopRwG8D+CqAvOVnA7v3IAeOXXI\nCBQvZzQarQuHB9tUtX8u9v2pmi45rWScpy6E+BqA7yfdvUFK+XaKh9cC6BdCtMdvDwC4DMDxglMS\nFSjeX5/xQdN2PX82c/GJVMhY1KWUBwEczHJbnwGYD+AeAB4ATwLI+B82sYfl8XgagT+8Wzrltnmf\nU/JkuN0M4N8dlCfV7WsMw9hRiuevra29OhAI/ENn56t/BgD33Xfjv/3VX4WvTvX45N+9iuf/6U97\nHrv//msv+BzXhx46/Ni3vhW4Tcf9mcvtYuxPt/3/zPV2JnkvEyCEaARwo3miVAjhBfBrACswXdQP\nSimvS/f9PFGqlg45dcgIFCdnODzYFgzO3WQWdZ9vCp2dQw8mrl2Tq4t5fxaDLjmLcqJUCLERwAMA\nvi6E+DEASCljmD5RehDAgfjXtafDLxnQI2cuGVX3n3NRjH1ZjLn4OvzOAea0Gxf0Isdx6+JdXIeG\nVCjWlMaLRmI/0Ml0yJltxlJf3l+sfZnLVMxs6PA7B5jTbizqREQuwqJuQZc+mw45s81Y6rVgdNiX\nAHOqpktOK1xPnRynkE8pIrrY8Ujdgi59Nh1y5pJRdf85FzrsS4A5VdMlpxUWdSIiF2FRt6BLn02H\nnDpkBJhTNea0F4s6EZGLsKhb0KXPpkNOHTICzKkac9qLRZ2IyEVY1C3o0mfTIacOGQHmVI057cWi\nTkTkIizqFnTps+mQU4eMAHOqxpz2YlEnInIRFnULuvTZdMipQ0aAOVVjTnuxqBMRuQiLugVd+mw6\n5NQhI8CcqjGnvVjUiYhchEXdgi59Nh1y6pARYE7VmNNeLOpERC7Com5Blz6bDjl1yAgwp2rMaS8W\ndSIiF2FRt6BLn02HnDpkBJhTNea0F4s6EZGLsKhb0KXPpkNOHTICzKkac9qLRZ2IyEVY1C3o0mfT\nIacOGQHmVI057cWiTkTkIizqFnTps+mQU4eMAHOqxpz2YlEnInIRFnULuvTZdMipQ0aAOVVjTnux\nqBMRuQiLugVd+mw65NQhI8CcqjGnvVjUiYhchEXdgi59Nh1y6pARYE7VmNNeLOpERC4yK99vFEIs\nA/AogF9JKb+XcP+zAJYAOAfgWSnlPxcaspQ8Hk+jDu/gOuTUISPAnKoxp73yLuoAKgBsBXBt0v0G\ngG9KKU8WsG0iIspD3u0XKeUhAJE0X/bku12n0eWdW4ecOmQEmFM15rSX5ZG6EOJrAL6fdPcGKeXb\nab7lLICQErbDAAADmklEQVQXhBARAC1SyvcKzEhERFmyLOpSyoMADma7QSnlOgAQQlwDYBuA1Xmn\ncwBd+mw65NQhI8CcqjGnvTyGYeT9zUKIRgA3Jp4oTfhaA4AfSilFqu/t7u7O/4mJiC5iy5cvT9vi\nzruoCyE2ArgBgA/TM2Dujt/fAeDzmG7DrJVS9uX1BERElLOCjtSJiMhZePEREZGLsKgTEbkIizoR\nkYsUckVpXtItLxD/WgWAYwAekVLusjtbUpZ0yyD8I6aXQSgD8C0p5Qclipgp4woAW+I3t0gpXy9F\nvlSEELcBWAtgEsAmKeUbJY6UkhDicgB7Mf0a+Y2U8jsljpSWk1436TjpdZOKk18zpmz3YSmO1M3l\nBVL5WwD/G9NLDZRaypxSyr+VUl4PoBXAjKmcNpuRUQhRhulsK+PjASGEk67w/S6ml5a4AcBDJc6S\nyXYA/yClXObkgh7npNdNSg573VxAg9cMgOz3oe1FPd3yAkKIKgBfA/AzOGCZAYtlEIDpKZvjNsVJ\nKU3GRQCOSSlHpZSjAN4HcJXt4dL7DwB/DqAJwL+WOEtKQggvgC9JKQ+XOosVp71uslDy100KTn/N\nJMu4D4vWfsljeYF1AP4ngP9UrEyp5JHTdAeAncVJdaEcM9YC6BdCtMdvDwC4DMDxIkacIV1mAAcA\nNAMoB1DyVkGanG0A5gghfgrgUgBPSClfsj1cggz78waU4HWTThb/V2173eTAEa+ZHGTch0Ur6rks\nLyCEmAcgIKX8kRDivxcrUyq5LoMAAEKIrwP4rZSytzipLpRjxs8AzAdwD6aP3J4EcKZI0dJKlVkI\n8UcAmqSUfxm//WshxKH40VFJpMk5C9Mv7FsAeAG8JYR4zYE55wFYJqV82O7XTTqZ/q/a/brJgSNe\nM9nIZh+WavZL8p+J12H6yGgfpvuD3xJCfNn+WDPM+HNWCPGnAP5cSrmjBHlSSc74PoDFCbcXOWhR\nNS/iBxLxnmUlHNgHllJOAvgIgE9KOQ5grMSR0nHq62YGB75uEjn5NXNetvvQ9itK0y0vkPD12wFU\nSymftDVYkgzLIHyA6Rf8FID/Zy5g5rCMKwH8IP6w1vjRkyMIIe4HEMD0AUWHlPLZ0iZKTQhxBYB/\nBDAPgJRSOq1lcAGnvG7ScdLrJhUnv2ZM2e5DLhNAROQivPiIiMhFWNSJiFyERZ2IyEVY1ImIXIRF\nnYjIRVjUiYhchEWdiMhFWNSJiFzk/wOteHEJuaJD9wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x112e12910>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "centroids = np.array(map(lambda cent: cent[1], res[1]))\n",
    "\n",
    "plt.scatter(data[:, 0], data[:, 1])\n",
    "plt.scatter(centroids[:, 0], centroids[:, 1], c='r', s=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x113040d90>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAEBCAYAAACHTjUfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X1wW+WdL/CvrMSO7UBi4/SqU4rDliSeirvLzO5MU1Bm\nzU2aLIO7NHD1NDt3gVsGll2yTeymbYBNGlxzSSEBJ3DD3S6QsIQh7uMNtBRanMRAK5Fspnvn7uXi\nxU2A4MCUlATJdvwSv8jn/mGdVJElHb08OjrPyfcz88xEsnz09XH00/HvPOeRxzAMEBGRO5SVOgAR\nEanDok5E5CIs6kRELsKiTkTkIizqREQuwqJOROQis/L9RiHEMgCPAviVlPJ78ftWANgSf8gWKeXr\nhUckIqJsFXKkXgFgq3lDCFEGoBXAyvh4QAjhKSweERHlIu+iLqU8BCCScNciAMeklKNSylEA7wO4\nqsB8RESUg7zbLynUAugXQrTHbw8AuAzAcYXPQUREGags6p8BmA/gHgAeAE8COKNw+0REZKHQop7Y\nM38fwOKE24uklO+l+8bu7m4uOkNElIfly5enPV9ZyOyXjQBuAOATQlwqpbxbCNEK4GD8IQ8UEoyI\niGayOiD2lGqVxu7ubkOHou7xeBoNw3iz1Dms6JBTh4wAc6rGnGpZ1U5efERE5CIs6hZ0eOcG9Mip\nQ0aAOVVjTnuxqBMRuQiLugWPx9NY6gzZ0CGnDhkB5lSNOe3Fok5E5CIs6hZ06bPpkFOHjABzqsac\n9mJRJyJyERZ1C7r02XTIqUNGgDlVY057sagTEbkIi7oFXfpsOuTUISPAnKoxp71Y1ImIXIRF3YIu\nfTYdcuqQEWBO1ZjTXizqREQuwqJuQZc+mw45dcgIMKdqzGkvFnUiIhdhUbegS59Nh5w6ZASYUzXm\ntBeLOhGRi7CoW9Clz6ZDTh0yAsypGnPai0WdiMhFWNQt6NJn0yGnDhkB5lSNOe3Fok5E5CIs6hZ0\n6bPpkFOHjABzqsac9mJRJyJyERZ1C7r02XTIqUNGgDlVY057sagTEbnIrFIHcDpd+mw65NQhI8Cc\nqmWTMxqN1vX0eNcDgN8f21lTU3Om6MGS6LI/rbCoE1FJRaPRuq6u2W+2tFT5AaC9fWT1qlXRxlIU\ndjdg+8WCLn02HXLqkBFgTtWscvb0eNe3tFT5T50qw6lTZWhpqfKbR+120mV/WuGROhHlzAntEkqN\nR+oWdOmz6ZBTh4wAc1ox2yXB4NxNweDcTV1ds9+MRqN16R5vldPvj+1sbx/p8fmm4PNNob19pMfv\nj+1UHtyCLr93KzxSJ6KcJLZLAKClpcrf2Tm0PhDA5ny2V1NTc2bVqmhjZ+eQsiP/i/kvCR6pW9Cl\nz6ZDTh0yAsypWjY5a2pqzgQCl24OBC7drKKg5/KXRC45dcAjdSLKid8f27lr1/B/DYVmNQDAsmWT\nvaVol6Sj+i8J3bCoW9Clz6ZDTh0yAsxpytTCmJiAp6OjAgCwdOmkp5Q5VdElpxW2X4hohkwtjJ4e\n7/rm5uol5hTE5ubqJaWYgpiOU068lgqLugVd+mw65NQhI8CcgNq548XMGY1G68LhwbZweLDNfNOZ\nPvE60djZOfRgZ+fQg6tWTWR1IZMuv3crbL8QUU7iR8KrE64AVXYknMuslUxXok6feL04eugzGIah\ndASDwWeDweCRYDD4RjAYvD3d4w4dOmSofm4ODg41IxKJ1O3bd/Ydny9m+HwxY9++s+9EIpG6xK+H\nQgNtodBAW+L9xXzO5BEKDbT5fDEDMAzAMHy+mBEKDbTl87yqf5ZiDqvaWYwjdQPAN6WUJ4uwbSKy\ngdXc8VyPhFMdgZv3eb1GpdcLjI+XfcXuWStuXHemWO2XjGfDdeLxeBoNDc6K65BTh4wAc5pUtDCm\ni+axXV7vl69ft656AXC+cN7c1TX7RbOYbtw4ilOncisbyW2gxx8fPu33x/bmsg03Tn8sxonSswBe\nEEL8XAhxVRG2T0QaMI+Cjx79Y7FuXfWCC0+6lj2TeCL24YcrMTrqwcaNo8h21kr8hOjNu3cPnV6z\nZgw/+EHlgq6u2S9mc6GRqxWr7xMMBq8JBoMvZeoLAWg0bwNo5G3e5m3n3A4EAjeZveZAIHBTrt+/\ne3foOZ8vZtx556iR3Pt+9dUTbyffd+edo0ZDw4QRCvWHdu8OPRcIBG6yer50ffVsf950fXwn7P90\nt6166mm/UOgIBoMNwWBQpvs6T5RycDh35HrSMtUwC25tbczYtm3YSNrW4sTtt7cPGQ0NEzk/j4qT\npW47UeoxDAMqCSE6AHwe022YtVLKvlSP6+7uNpYvX+743jv7q+rokBFgTgAIhwfbgsG5m8xes883\nhc7OoQcDgUtzOjlqnoScP38Kjz02cnr+fOO5hoapHyWeKJ09e+qSL8Q++sqck+99vmJq9GPjiive\niPn9W2tqakZyeQ5genpltvPSk+nye7eqncpPlEop16jeJhE5U6Z55eYMmoceOvzYokV/3Jfq635/\ndOuscLi76tvfXurt6wOA+lh9/XUjTzyxPBoIrABQlWneejFWeNSeU/+E4ODgKN3Ipv2iokUzEAq1\nTdbXG+f7J/ExWV9v9B8+vK3Q7btxWNVOLhNARDNkc6m9iqUEyj78cFn8CP0C3r4+jH105q7W1jn+\n8XGgqWkcR496/cePl91b4I/meizqFnRZD0KHnDpkBJgTUPMhE+a6LHv2hJ/LZ5rhu8cr5q1bdw6b\nNo3ilVfK0dFRgb6+stuKNWVRl9+7FRZ1IrpAth8y4ffH9ra3D4/9YV758Jh58U/iNu6//9pb021j\nauHCUKy+fkaGWH09pr70R+jrK8Mjj1TC/Gtg3brqBU5aEdKJWNQtGBqcDQf0yKlDRoA5s22r9PR4\nb21traxoahpHU9M4WlsrK3p6vLdmu41oNFrX4/3Ps6K7nvo4sbDH6uvxxtrnccd9V+JLX5oqxo+Y\nki6/dytcpZGI8tbfX4ann54DYHraY7YSpyLOmfNf8NP/1fXp5YPvjv17b9UXX/9oCdq3XIF58wws\nXBg7s2PH8GfNzdVLgNQrQqr6PFLXfK6pU8/gOmUg4aouJw8dcuqQkTmzn9USiUTqOjoGe83HdXQM\n9pqPS97Giy8OHH/rrf4joVB/KBKJLE510dDhw/3bE79HysFPI5HI4kwXB6mYgWNuZ8+eT07oMNOm\nFKs0EpHGcpn7PXs2jDVrxs7/O9U2fv/741ePjy++4ZZb5l4FADt3Dv/HlVdOPZe8rVjMMxqfcTPj\nedMtsKVqQa6eHu/6++773EI3LOzFom7B0KTPpkNOHTICzAlkt0JjT493/dq11Q1mIXzttcmGp54a\nfgkYfNMsyIEANofDV4WCweoK83Hr11d79+49+80dO4Z/m9xWUfXhFq5ppeSBJ0qJqGC1tVO4++4x\nBIOXBDLNmDG9+mp51ZVXTr2S60fOJUvzeaR7s5m9k8V29PxcU6f2hZwycJH3Vy+2jMyZ/UjsZzc3\nj8xYidFcWOuOO+7465/8ZHDS7Fdv2zYcX40x908pSpcjseee7yJfiatSOrWfbhjsqRNRkVzYezca\nOzoqAqket3v37o+3b99+3Z49Q693dc2ueuaZcmzZck7ZkfDMls1gXtsJh8MDVguWadHWceq7DQcH\nhz4j27Vi7DgSVjUjxq7t5jpsX3o3W7osvUtE2XHSUWwxsqhYjlgF25fedRtd1ljWIacOGQHmzFe6\nmSulyJnPLBqn7c98cfYLEVEWdJkhwyN1C7q8c+uQU4eMAHOq5pacunwgB4s6EV1UCum3q7o4qpjY\nfrGgyxrLOuTUISPAnKo5KWemZYWdlLMQLOpEpC3zgzjC4cG2bD48Q8WnNTkd2y8W3NIPdAIdMgLM\nqVqxciYu3wsA7e0jq1etiua13ACgz/60wiN1ItJSPkfdusxgKQSLugVd+mw65NQhI8CcqjkpZ6YP\n1HZSzkKw/UJEWoofda9OaL9kddStwwyWQrCoW9Clz6ZDTh0yAsypWrFyqp43rsv+tMKiTkTacvtR\ndz7YU7egS59Nh5w6ZASYUzXmtBeLOhGRi7CoW9Clz6ZDTh0yAsypGnPai0WdiMhFWNQt6NJn0yGn\nDhkB5lSNOe3Fok5E5CIs6hZ06bPpkFOHjABzqsac9mJRJyJyERZ1C7r02XTIqUNGgDlVY057sagT\nEbmI8mUChBArAGyJ39wipXxd9XPYSZc+mw45dcgIMKdqzGkvpUfqQogyAK0AVsbHA0IIj8rnICKi\n9FS3XxYBOCalHJVSjgJ4H8BVip/DVrr02XTIqUNGgDlVY057qW6/1ALoF0K0x28PALgMwHHFz0NE\nRCmoLuqfAZgP4B4AHgBPAsh7fWMn0KXPpkNOHTICzKkac9pLdfvlfQCLE24vklK+l+7BiX/ueDye\nRt7mbd7mbd62vp2JxzCMbB6XNSHESgA/iN9slVIeTPW47u5uY/ny5Y4/ierxeBp1eAfXIacOGQHm\nVI051bKqncqnNEopDwA4oHq7RERkjRcfWdDhnRvQI6cOGQHmVI057cWiTkTkIizqFrI9OVFqOuTU\nISPAnKoxp71Y1ImIXIRF3YIufTYdcuqQEWBO1ZjTXizqREQuwqJuQZc+mw45dcgIMKdqzGkvFnUi\nIhdhUbegS59Nh5w6ZASYUzXmtBeLOhGRi7CoW9Clz6ZDTh0yAsypGnPai0WdiMhFlC/o5Ta69NmK\nnTMajdb19HjXA4DfH9tZU1OT8zr53JdqMadauuS0wqJOlqLRaF1X1+w3W1qq/ADQ3j6yetWqaGM+\nhZ2IiovtFwu69NmKmbOnx7u+paXKf+pUGU6dKkNLS5XfPGrPBfelWsypli45rbCoExG5CIu6BV36\nbMXM6ffHdra3j/T4fFPw+abQ3j7S4/fHdua6He5LtZhTLV1yWmFPnSzV1NScWbUq2tjZOVTQiVIi\nKj4WdQsej6cxEom8U+jMj1zlOtuk2J+vWFNTcyYQwOZCtqHLZ0Ayp1rMaS8WdQuBQGCe3TM/8plt\nEggE5oXDg20Aj6SJLmqGYZRkHDp0yCjVc+cyQqGBNp8vZgCGARiGzxczQqGBNic9ZyQSqdu37+w7\nPl/M8Plixr59Z9+JRCJ1pd53HBwc6odV7eSJ0iKJRqN14fBgWzg82BaNRuuK+VzmlMPxcaCpaRxH\nj3r9x4+X3VvM5yQiZ2JRt3DffTf+W64zP8z2STA4d1MwOHdTV9fsN3Mp7MmzTXbtGu71eo3KTG8Q\n8+dP4b77zuGVV8rR0VGBkyfL/i4ajS7O9ectJl3mATOnWsxpM6f+CeGUAaAxEonUhUIDbaHQQFs2\nbY18WzZJz7M4FBpoO3q0f3tHx2BvptZKJBKpe/nlM9Hk5/zFLwY+dVIbBkBjqTMwJ3PqntOqdvJE\nqQUjfja80JkfVlKfHJ1o7Onxrm9url5y6tT0H1UtLVX+zs6h9Yl5ampqztTVDTwDYEPiNg8cmL3g\nssuMe8fHB0eB0p9ANTSZWcCcajGnvdh+KQK/P7a3vX147A8tm+Exvz+2N9P3FHopfkPD1I8ef3z4\ntPmcGzacw5EjXvT1ld2WbxuIiPTDom4hlz6beXK0p6fsmdbWyoqmpnE0NY2jtbWyoqfHe2um7y0v\nNyrXrBnDnXeeQ23t1Pn7s72as7a29uoVKya//vzzZ0d++MMR/Mu/zMKWLedOr1tXvSDfN4pcTvZm\n81hdepbMqRZz2ovtF0US2ydr1oyhv78MTz89BwDg801Zfu+JE7OaOjoqAACbN4/iiiumes12SfLV\nnABGBsPhtrIPP1wGAFMLF4a+8Y1vvN3VNXtLS0tVFQA8/vjw6dpaoxPAPYX+PEDmufJcxZHIQZza\n7NdtJJ4cra2NGe3tQ4Z5crOjY7D36NH+7elOtKY6sXr4cP/2VM8TiUSqBl9++chkfb1hfsNkfb1x\n+pUDHy1cOHnBNn7zm+iuxJOsHR2DvdmeOE19src/lG3+Ys/l5+C4WAdPlJZAJFKGH/+4Ap2dZ8Ne\nL45+8knZjTfddMkGIPuj2FjMMwr8YbmA8nKjMhYDvhD76Ctf+Pa3l3r7+s4/1tvXh5q1d13+N994\nE/fvXHj+/mjUI8rLjf41a8YAALNnwyjk59q/vzzw8cexN3kUTuRgTn23ccpAltOc0l3Vmc1RbLrv\nNe9vaJg4f+R/6IEu4/zGksbJ518dN7exbduwsXHjsJHvEXRypm3bho3a2lhO+fPdl6UezMmcTh48\nUrdJ+pUMB/P+3nB4sK2lpcrf1DSOhx+uxPg4cNll6Q+2q2aNn1qzZuyLQ0MebN06B7fdNqbg5zn7\n0v795YGtW+cgEilLeX6AqzgSOYhT323cMgpZl8U8yr/zzlGjoWHC2LZt2Nja/IGR2E9P7Kv3Hz68\nLfG59u8ffNfqwqVi5ufg4FA/rGqnxzAKarPmrbu721i+fLmnJE9us3w/tNmcVdLaOsff2jqK9eur\nMTDgwcutb+H6XX8Ns68eq6/HyBNPHJkMBFYAqEp8LmB6Dnyuz60iPxGpZ1U7WdQtZLPGcjGLnrnt\nsjJjxS23XLL01KkyVFYa2Pw3fbhr6f/pmzM5/OHUwoW/vua229764IMPulQ9bzHosl41c6rFnGpZ\n1U721Atk1xztWbPw1o4dwzXNzdVLAOCq6y477V3RuHJuTc0xADhx4kRjPtvlUTiRyzi1L6TLKOYc\n7eR+9ssvDxz75S8HzjQ3jxgNDRPG/v2D7x4+nH7+e67bZ7+cg8P5w/bZL0KIZwEsAXAOwLNSyn9W\n/RwXi8T1YADg9ddnL+roqMCpU2WorZ3CyZNlDWvXVjcA+f2FkLz9lpYq/4svnr33q1/Fd4vx8xBR\n8RVj7RcDwDellNe7oaBbrQeR7dosqt188/Q0x3zXdQFSrzfT3++5rViLfumytgZzqsWc9ipWT93x\nJ0BVKeYc7fgbxmqzX79s2WTv0qWTnubm6iVz5xZ2gjvVejNeL/Cd71QteOqpkfXFXmqYiIoj76Iu\nhPgagO8n3b0BwFkALwghIgBapJTvFZCv5IwszobX1NScKUYRTLOYFzo7h9aXlxuVS5dONpknTnP9\nCyF5nfaHH65EMDi9EFmxZLMvnYA51WJOe+Vd1KWUBwEcTPGldQAghLgGwDYAq/N9Dkr9hmHeXrQo\n+qNMfyHkOrPF6839zYGIHKZYZ2iDwWBDMBiUmc7gImGtBQCNTrxt3ueUPBluNyfeDgQCN+3Z88kJ\nc2bLnj2fnAgEAjel+/oLL3wW3bfvX39izn4pUt5mB+2vtLeTf/elzsP9yf2ZeNv2K0qFEB0APo/p\nNsxaKWVfqse56eIjJ0jOGQ4PtgWDczeZ7RWfbwqdnUMPBgKXnj/qt3uOuq770qmYUy1dctp+8ZGU\nco3qbZaSk3/JiUU5Eonk3DIp1rmAdJy8LxMxp1rMaS9+nJ2mzCtZ033+aKmmWhJRabGoW3Dq3FWr\nD6qenjkz0djZOfRgZ+fQg6tWTZT8gy2cui+TMadazGkvrv3iUlzTJXfmPtu9O1QfjUbf4T4jLWU6\ni1rM4Za1X0o1Mq3bwjVd1O5PDg4nDavayfaLpjK1V6xaMzQT9xm5BYu6BSf32aZnr1y6ORC4dHNt\nbe3Vpc5jxcn7Uke67E/mtBeLugtx5kvuuM/ILfjJRy7FE6W54z4jHfCTjy5Sdl9Y5AbcZ+QGbL9Y\n0KXPpkNOHTICzKkac9qLR+rkWmyn0MWIRd2Cocl6EDrktDNjIR8IrsO+BJhTNV1yWmH7hVzJat55\nNBqtC4cH28LhwTbVH99XzG0TWWFRt6BLn02HnE7JaLUYWiE5rbatklP2pxXmtBeLOpVMuiNaFUe6\nmeadF/PqUV6ZSqXGnroFXfpsOuRMzJiu5w0A+fbCE+X6geBer1GZKqeTMadauuS05NRFaTjcPUKh\ngTafL2YAhgEYhs8XM0KhgbZ096faRiQSqTO/JxKJLE74d8aFuCKRSF1Hx2CvuXhXe/uQsX//4Lsq\nFvDiwmAcxR5c0KtAuvTZdMipMuPM3vWst++6qyqrPnZNTc2ZK6+cemXNmjE0NY2jra0Sa9dWN5ht\nkkJy2rmOvQ6/c4A57cb2C5VEvOe9OqHNcr7nne7+RIm9awBoaamuaGoax9NPz0FLS5W/s3NofeLV\noclz1sfHvaMdHRVI/AxXUyAQmBcOD7aZj82n9cMrU6lUWNQtGJr02UqdM5sLfRIzZup559ILzzbb\nzD79xM2p3jyi0Wjd2rW//B/BYGE9fTuU+neeLea0mVP7Qhz6jFL0kWc+5+C5hoaJlB8YEgr1H0nV\np0/dk+8PZdvT5+AoxbCqnTxSt+DxeBoNDd7BS5lzZitkZvtDdcYUR/p7L7985Nb4v3cCGBk4cmT7\n0AeRv5s36K1av2YJfvjjhRgdnV7czus1KhP+stjb1TX7xZaWKv+aNWMq4tmC/zfV0iWnFRZ10lZy\n79rvj+7s6fGu7+0t23D1Z6EVVX//9382r68PXwTw5fp6XPvQ87h951fxyCMjn33ySdmNa9dWNwDA\n7t1Dd7e0VC04daoMzz1Xgc2bR/Hww9MzHLmuOmnHqX9CcOgzit1+iUQidUeP9m8PhfpDhw/3b0+1\n7cQMW5s/MCbr643zPZT4mKyvN9568Zhx5Ej0gnZMc/OIkXi7oWHCCIX6Q9lMj+TgsHuw/UJFl+uF\nPrmIRqN1b7wxK3TyZFlD/Og5sGPHcNPKldG/7Onxnm+39PaW3Wu2gP6k6j14+/pmbMvb14crRo+N\n/K7sc28BWGre/9prs/H448On162rXgAAW7ac6/H7p1abP4Mdqz1yRUlSxqnvNk4ZABpLncEtOfPJ\nGAoNtKU6kpZy8FPzL4OOjsHeAwf6T5uP+cX9B4zko3RznOl46eThw/3b9+8ffDfpL4vzFy8FAoGb\nzOe34yRwvs+hw++cOdUPXnxErvMXfzGBdeuqF5jrqzQ3Vy/p7p5dt2HDOfh8U/i/I1chVl8/4/ti\n9fX4pyPXfPHmmy/ZMDEBz89+dvbRhAuEjpkf4h0OhwfM77FjLReuF0MqsahbMDQ5G65Dznwy+v2x\nncuWTfZu3DgKc3GulSsnRpMfNzbmwdatc9DUNI6h6s/h9088O5lY2GP19Tj63b1o+6f6828E4+Oe\n0UDg0s3JrQ4d9iXAnKrpktMKizo5Wk1NzZnrr59cFghMPPn882dHbr/9HFpb51S2tw+PmUV+x47h\n3y5bNtlbXg688ko5rr7a6KkM/Mmf/u6Fnx/59CcvD3zW8dJHv3vh50f+26PXnp/SmK1Mqz2qYsdz\n0EXEqX0hpwxo0mfTIWchGZMX+kqeoZJ0IVHG2TFWfevknFbbVjHyeQ4dfufMqX5w9gu5Un9/GQDP\nm4HApefnqWdab6WQGTp2rOXC9WJIFU/8Hcp23d3dxvLly3P7W5hcy2pKX4r1W3qKuQIikVNZ1U4e\nqVPJZfMh0YlH2uXlRmUsNj1rxO+PZjzi5vxvuug4tS/klAFN+mw65EyXMdcPxsi2N875384YzKl2\ncJ46uUouc7o5/5suRizqFgxN5q7qkDNdRqdN6dNhXwLMqZouOa2wqFPJ5fIRcLm8ASQ/dteu4V6v\n16gMhwfbMn3cHZHO8pr9IoRYBuBRAL+SUn4v4f4VALbEb26RUr6ebhu6zH7RZY1lHXKqypjLyU/z\nseXlRuWJE2VNzc3VS4DMs2d02JcAc6qmS06r2pnvkXoFgK2JdwghygC0AlgZHw8IIRxftEk/03O6\np9dpsZrNYj52fNwz2txcvYT9dXK7vIq6lPIQgEjS3YsAHJNSjkopRwG8D+CqAvOVnA7v3IAeOXXI\nCBQvZzQarQuHB9tUtX8u9v2pmi45rWScpy6E+BqA7yfdvUFK+XaKh9cC6BdCtMdvDwC4DMDxglMS\nFSjeX5/xQdN2PX82c/GJVMhY1KWUBwEczHJbnwGYD+AeAB4ATwLI+B82sYfl8XgagT+8Wzrltnmf\nU/JkuN0M4N8dlCfV7WsMw9hRiuevra29OhAI/ENn56t/BgD33Xfjv/3VX4WvTvX45N+9iuf/6U97\nHrv//msv+BzXhx46/Ni3vhW4Tcf9mcvtYuxPt/3/zPV2JnkvEyCEaARwo3miVAjhBfBrACswXdQP\nSimvS/f9PFGqlg45dcgIFCdnODzYFgzO3WQWdZ9vCp2dQw8mrl2Tq4t5fxaDLjmLcqJUCLERwAMA\nvi6E+DEASCljmD5RehDAgfjXtafDLxnQI2cuGVX3n3NRjH1ZjLn4OvzOAea0Gxf0Isdx6+JdXIeG\nVCjWlMaLRmI/0Ml0yJltxlJf3l+sfZnLVMxs6PA7B5jTbizqREQuwqJuQZc+mw45s81Y6rVgdNiX\nAHOqpktOK1xPnRynkE8pIrrY8Ujdgi59Nh1y5pJRdf85FzrsS4A5VdMlpxUWdSIiF2FRt6BLn02H\nnDpkBJhTNea0F4s6EZGLsKhb0KXPpkNOHTICzKkac9qLRZ2IyEVY1C3o0mfTIacOGQHmVI057cWi\nTkTkIizqFnTps+mQU4eMAHOqxpz2YlEnInIRFnULuvTZdMipQ0aAOVVjTnuxqBMRuQiLugVd+mw6\n5NQhI8CcqjGnvVjUiYhchEXdgi59Nh1y6pARYE7VmNNeLOpERC7Com5Blz6bDjl1yAgwp2rMaS8W\ndSIiF2FRt6BLn02HnDpkBJhTNea0F4s6EZGLsKhb0KXPpkNOHTICzKkac9qLRZ2IyEVY1C3o0mfT\nIacOGQHmVI057cWiTkTkIizqFnTps+mQU4eMAHOqxpz2YlEnInIRFnULuvTZdMipQ0aAOVVjTnux\nqBMRuQiLugVd+mw65NQhI8CcqjGnvVjUiYhchEXdgi59Nh1y6pARYE7VmNNeLOpERC4yK99vFEIs\nA/AogF9JKb+XcP+zAJYAOAfgWSnlPxcaspQ8Hk+jDu/gOuTUISPAnKoxp73yLuoAKgBsBXBt0v0G\ngG9KKU8WsG0iIspD3u0XKeUhAJE0X/bku12n0eWdW4ecOmQEmFM15rSX5ZG6EOJrAL6fdPcGKeXb\nab7lLICQErbDAAADmklEQVQXhBARAC1SyvcKzEhERFmyLOpSyoMADma7QSnlOgAQQlwDYBuA1Xmn\ncwBd+mw65NQhI8CcqjGnvTyGYeT9zUKIRgA3Jp4oTfhaA4AfSilFqu/t7u7O/4mJiC5iy5cvT9vi\nzruoCyE2ArgBgA/TM2Dujt/fAeDzmG7DrJVS9uX1BERElLOCjtSJiMhZePEREZGLsKgTEbkIizoR\nkYsUckVpXtItLxD/WgWAYwAekVLusjtbUpZ0yyD8I6aXQSgD8C0p5Qclipgp4woAW+I3t0gpXy9F\nvlSEELcBWAtgEsAmKeUbJY6UkhDicgB7Mf0a+Y2U8jsljpSWk1436TjpdZOKk18zpmz3YSmO1M3l\nBVL5WwD/G9NLDZRaypxSyr+VUl4PoBXAjKmcNpuRUQhRhulsK+PjASGEk67w/S6ml5a4AcBDJc6S\nyXYA/yClXObkgh7npNdNSg573VxAg9cMgOz3oe1FPd3yAkKIKgBfA/AzOGCZAYtlEIDpKZvjNsVJ\nKU3GRQCOSSlHpZSjAN4HcJXt4dL7DwB/DqAJwL+WOEtKQggvgC9JKQ+XOosVp71uslDy100KTn/N\nJMu4D4vWfsljeYF1AP4ngP9UrEyp5JHTdAeAncVJdaEcM9YC6BdCtMdvDwC4DMDxIkacIV1mAAcA\nNAMoB1DyVkGanG0A5gghfgrgUgBPSClfsj1cggz78waU4HWTThb/V2173eTAEa+ZHGTch0Ur6rks\nLyCEmAcgIKX8kRDivxcrUyq5LoMAAEKIrwP4rZSytzipLpRjxs8AzAdwD6aP3J4EcKZI0dJKlVkI\n8UcAmqSUfxm//WshxKH40VFJpMk5C9Mv7FsAeAG8JYR4zYE55wFYJqV82O7XTTqZ/q/a/brJgSNe\nM9nIZh+WavZL8p+J12H6yGgfpvuD3xJCfNn+WDPM+HNWCPGnAP5cSrmjBHlSSc74PoDFCbcXOWhR\nNS/iBxLxnmUlHNgHllJOAvgIgE9KOQ5grMSR0nHq62YGB75uEjn5NXNetvvQ9itK0y0vkPD12wFU\nSymftDVYkgzLIHyA6Rf8FID/Zy5g5rCMKwH8IP6w1vjRkyMIIe4HEMD0AUWHlPLZ0iZKTQhxBYB/\nBDAPgJRSOq1lcAGnvG7ScdLrJhUnv2ZM2e5DLhNAROQivPiIiMhFWNSJiFyERZ2IyEVY1ImIXIRF\nnYjIRVjUiYhchEWdiMhFWNSJiFzk/wOteHEJuaJD9wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x112e74a10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "init_centroids = test_rows.takeSample(False, 3)\n",
    "res = Kmeans(test_rows, init_centroids, 10)\n",
    "\n",
    "centroids = np.array(map(lambda cent: cent[1], res[1]))\n",
    "\n",
    "plt.scatter(data[:, 0], data[:, 1])\n",
    "plt.scatter(centroids[:, 0], centroids[:, 1], c='r', s=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(PythonRDD[48] at RDD at PythonRDD.scala:43,\n",
       " [(0, array([-6.58196786, -8.17239339])),\n",
       "  (1, array([-10.04935243,  -3.85954095])),\n",
       "  (2, array([-1.4710815 ,  4.33721882]))],\n",
       " [619.94523463088854,\n",
       "  213.8268574473714,\n",
       "  108.56785435636178,\n",
       "  108.56785435636178])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## All Together Now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk, string\n",
    "\n",
    "def tokenize(text):\n",
    "    tokens = [] \n",
    "    \n",
    "    for word in nltk.word_tokenize(text):\n",
    "        if word \\\n",
    "            not in nltk.corpus.stopwords.words('english') \\\n",
    "            and word not in string.punctuation \\\n",
    "            and word != '``':    \n",
    "                tokens.append(word)\n",
    "    \n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sc.stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'local[*]'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# run on all cores\n",
    "sc = ps.SparkContext()\n",
    "sc.master"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# parse input essay file\n",
    "essay_rdd = sc.textFile('file:///Users/jonathandinu/spark-ds-applications/data/donors_choose/essay_parse.json')\n",
    "row_rdd = essay_rdd.map(lambda x: json.loads(x))\n",
    "\n",
    "# tokenize documents\n",
    "tokenized_rdd = row_rdd.filter(lambda row: row['essay'] and row['essay'] != '') \\\n",
    "                       .map(lambda row: row['essay']) \\\n",
    "                       .map(lambda text: text.replace('\\\\n', '').replace('\\r', '')) \\\n",
    "                       .map(lambda text: tokenize(text))\n",
    "\n",
    "# compute term and document frequencies\n",
    "term_frequency = tokenized_rdd.map(lambda terms: Counter(terms))\n",
    "\n",
    "doc_frequency = term_frequency.flatMap(lambda counts: counts.keys()) \\\n",
    "                             .map(lambda keys: (keys, 1)) \\\n",
    "                             .reduceByKey(lambda a, b: a + b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "num_features = 10000\n",
    "\n",
    "top_terms = doc_frequency.top(num_features, key=lambda a: a[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "total_docs = essay_rdd.count()\n",
    "idf = map(lambda tup: (tup[0], math.log(float(total_docs) / (1 + tup[1]))), top_terms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "broadcast_idf = sc.broadcast(idf)\n",
    "\n",
    "def vectorize(tokens):\n",
    "    word_counts = Counter(tokens)\n",
    "    doc_length = sum(word_counts.values())\n",
    "    \n",
    "    vector = [word_counts.get(word[0], 0) * word[1] / float(doc_length) for word in broadcast_idf.value]\n",
    "    return np.array(vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bag_of_words = tokenized_rdd.filter(lambda x: len(x) > 0).map(vectorize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PythonRDD[30] at RDD at PythonRDD.scala:43"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bag_of_words.persist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "top_n = 10\n",
    "summary = bag_of_words.map(lambda x: map(lambda idx: broadcast_idf.value[idx][0], np.argsort(x)[::-1][:top_n]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[u'SAT',\n",
       "  u'preparation',\n",
       "  u'tutors',\n",
       "  u'classes',\n",
       "  u'Unlike',\n",
       "  u'Regents',\n",
       "  u'cash',\n",
       "  u'term',\n",
       "  u'11th',\n",
       "  u'districts'],\n",
       " [u'doll',\n",
       "  u'pregnancy',\n",
       "  u'baby',\n",
       "  u'Think',\n",
       "  u'effort',\n",
       "  u'Over',\n",
       "  u'simulated',\n",
       "  u'alter',\n",
       "  u'prevention',\n",
       "  u\"''\"],\n",
       " [u'frequently',\n",
       "  u'rampant',\n",
       "  u'team',\n",
       "  u'Walk',\n",
       "  u'evening',\n",
       "  u'views',\n",
       "  u'runs',\n",
       "  u'entertainment',\n",
       "  u'Seeing',\n",
       "  u'NY'],\n",
       " [u'SAT',\n",
       "  u'P',\n",
       "  u'college',\n",
       "  u'History',\n",
       "  u'advisory',\n",
       "  u'professions',\n",
       "  u'talks',\n",
       "  u'counselor',\n",
       "  u'aside',\n",
       "  u'II'],\n",
       " [u'P',\n",
       "  u'seniors',\n",
       "  u'contemporary',\n",
       "  u'graduating',\n",
       "  u'graduation',\n",
       "  u'Bronx',\n",
       "  u'Richard',\n",
       "  u'Memory',\n",
       "  u'Ralph',\n",
       "  u'gift'],\n",
       " [u'weather',\n",
       "  u'television',\n",
       "  u'India',\n",
       "  u'P.S',\n",
       "  u'player',\n",
       "  u'reports',\n",
       "  u'grant',\n",
       "  u'examples',\n",
       "  u'funded',\n",
       "  u'equipment.'],\n",
       " [u'Richard',\n",
       "  u\"'ve\",\n",
       "  u'novels',\n",
       "  u'flame',\n",
       "  u'God',\n",
       "  u'Chocolate',\n",
       "  u'Following',\n",
       "  u'fan',\n",
       "  u'Eyes',\n",
       "  u'copies'],\n",
       " [u'geography',\n",
       "  u'slide',\n",
       "  u'assigned',\n",
       "  u'themes',\n",
       "  u'9th',\n",
       "  u'theme',\n",
       "  u'historical',\n",
       "  u'Roughly',\n",
       "  u'understanding',\n",
       "  u'Humanities'],\n",
       " [u'slavery',\n",
       "  u'Academy',\n",
       "  u'article',\n",
       "  u'...',\n",
       "  u'--',\n",
       "  u'born',\n",
       "  u'10th',\n",
       "  u'New',\n",
       "  u'North',\n",
       "  u'inside'],\n",
       " [u'P',\n",
       "  u'error',\n",
       "  u'trial',\n",
       "  u'Harlem',\n",
       "  u'attain',\n",
       "  u'listed',\n",
       "  u'boy',\n",
       "  u'learns',\n",
       "  u'thirty',\n",
       "  u'Academy'],\n",
       " [u'Bronx',\n",
       "  u'oxygen',\n",
       "  u'River',\n",
       "  u'river',\n",
       "  u'P',\n",
       "  u'water',\n",
       "  u'compare',\n",
       "  u'measurements',\n",
       "  u'land',\n",
       "  u'near'],\n",
       " [u'P',\n",
       "  u'Heritage',\n",
       "  u'Dictionary',\n",
       "  u'Pablo',\n",
       "  u'Man',\n",
       "  u'Heights',\n",
       "  u'owned',\n",
       "  u'PS',\n",
       "  u'forty',\n",
       "  u'Approximately'],\n",
       " [u'selection',\n",
       "  u'disappointed',\n",
       "  u'baskets',\n",
       "  u'eagerly',\n",
       "  u'Too',\n",
       "  u'stored',\n",
       "  u'West',\n",
       "  u'materials',\n",
       "  u'labeled',\n",
       "  u'Academy'],\n",
       " [u'He',\n",
       "  u'talent',\n",
       "  u'dramatic',\n",
       "  u'Bronx',\n",
       "  u'improvisation',\n",
       "  u'Economics',\n",
       "  u'twelfth',\n",
       "  u'tenth',\n",
       "  u'star',\n",
       "  u'remarkable'],\n",
       " [u'geography',\n",
       "  u'Egyptian',\n",
       "  u'documents',\n",
       "  u'scanner',\n",
       "  u'collect',\n",
       "  u'edit',\n",
       "  u'pyramid',\n",
       "  u'state-of-the-art',\n",
       "  u'Humanities',\n",
       "  u'excerpts']]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary.take(15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Kmeans on Essays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sample_data = bag_of_words.sample(False, 0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PythonRDD[32] at RDD at PythonRDD.scala:43"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_data.persist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# initialize centroids\n",
    "k = 100\n",
    "\n",
    "centroids = sample_data.takeSample(False, k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(PythonRDD[53] at RDD at PythonRDD.scala:43,\n",
       " [(0, array([ 0.0005783 ,  0.0032016 ,  0.00302903, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (78, array([ 0.00059769,  0.00379232,  0.00193364, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (39, array([ 0.00056981,  0.00327351,  0.00375755, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (40, array([ 0.00020996,  0.00342656,  0.00253888, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (1, array([ 0.00070413,  0.00337991,  0.00278605, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (79, array([  5.67428587e-04,   3.35750602e-03,   2.55920240e-03, ...,\n",
       "            4.38654131e-05,   1.09120055e-05,   8.17550213e-06])),\n",
       "  (80,\n",
       "   array([ 0.0005359,  0.0087459,  0.       , ...,  0.       ,  0.       ,  0.       ])),\n",
       "  (41, array([ 0.00063245,  0.00302752,  0.00235   , ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (2, array([  6.26325568e-04,   3.02990647e-03,   3.38240509e-03, ...,\n",
       "            0.00000000e+00,   6.79601550e-05,   3.70210961e-05])),\n",
       "  (81, array([  5.77263979e-04,   3.18622814e-03,   2.71257810e-03, ...,\n",
       "            0.00000000e+00,   2.99295726e-05,   0.00000000e+00])),\n",
       "  (42, array([ 0.00064761,  0.00416948,  0.00188924, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (3, array([ 0.00076453,  0.        ,  0.00184897, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (82, array([ 0.00055855,  0.00335821,  0.00208884, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (43, array([ 0.00073506,  0.00250052,  0.00250245, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (4, array([ 0.00053286,  0.00115949,  0.00128867, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (83, array([  6.19082134e-04,   3.43161652e-03,   1.99547094e-03, ...,\n",
       "            1.71337097e-05,   1.89952638e-05,   1.22850605e-05])),\n",
       "  (44, array([  5.89601467e-04,   3.75176131e-03,   1.96819622e-03, ...,\n",
       "            0.00000000e+00,   4.05572452e-05,   6.82928519e-05])),\n",
       "  (5, array([  6.20991572e-04,   3.55756522e-03,   2.09757112e-03, ...,\n",
       "            0.00000000e+00,   5.51378605e-05,   0.00000000e+00])),\n",
       "  (84, array([ 0.00051329,  0.00070629,  0.00183214, ...,  0.        ,\n",
       "           0.        ,  0.00686429])),\n",
       "  (45, array([ 0.00058371,  0.00263329,  0.0028977 , ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (6, array([ 0.00067788,  0.00302859,  0.00253372, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (85, array([  6.19468849e-04,   3.31190385e-03,   2.53333998e-03, ...,\n",
       "            1.48046467e-04,   0.00000000e+00,   6.94207526e-05])),\n",
       "  (46, array([ 0.00066356,  0.0014439 ,  0.00320953, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (7, array([ 0.00066106,  0.00656183,  0.00300223, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (8, array([  5.55042327e-04,   3.52646639e-03,   2.93078017e-03, ...,\n",
       "            1.18610848e-05,   3.51744190e-05,   0.00000000e+00])),\n",
       "  (86, array([ 0.00061014,  0.00408245,  0.00201029, ...,  0.        ,\n",
       "           0.00017474,  0.        ])),\n",
       "  (47, array([ 0.00057396,  0.00353696,  0.00248201, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (48, array([ 0.00056845,  0.00298502,  0.00143702, ...,  0.00185957,\n",
       "           0.        ,  0.        ])),\n",
       "  (9, array([  6.38024882e-04,   3.74281451e-03,   1.84901741e-03, ...,\n",
       "            3.20166683e-05,   0.00000000e+00,   0.00000000e+00])),\n",
       "  (87, array([  5.78446779e-04,   4.47018849e-03,   2.24665237e-03, ...,\n",
       "            4.32897146e-05,   0.00000000e+00,   0.00000000e+00])),\n",
       "  (88, array([ 0.00056662,  0.00332728,  0.00262316, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (49, array([ 0.00067039,  0.00315868,  0.00194504, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (10, array([ 0.00070184,  0.00275847,  0.00211094, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (89, array([ 0.00054666,  0.00398091,  0.00235092, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (50, array([ 0.00060655,  0.00188934,  0.00226047, ...,  0.00018931,\n",
       "           0.        ,  0.        ])),\n",
       "  (11, array([ 0.,  0.,  0., ...,  0.,  0.,  0.])),\n",
       "  (90, array([ 0.00062652,  0.00273665,  0.00295086, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (51, array([ 0.00061132,  0.0040377 ,  0.00206783, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (12, array([ 0.00068507,  0.00272029,  0.00164601, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (91, array([ 0.00059754,  0.00279001,  0.00183062, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (52, array([ 0.00061779,  0.00423335,  0.00205948, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (13, array([  5.24421543e-04,   3.47260283e-03,   2.55471048e-03, ...,\n",
       "            0.00000000e+00,   9.68969479e-05,   0.00000000e+00])),\n",
       "  (92, array([ 0.00079253,  0.00431136,  0.00239584, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (53, array([ 0.00059438,  0.00340048,  0.00185722, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (14, array([ 0.00054961,  0.00367539,  0.00298459, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (93, array([ 0.00066249,  0.00312638,  0.00186849, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (54, array([ 0.00060739,  0.00321073,  0.00209894, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (15, array([ 0.00058755,  0.0035551 ,  0.00208119, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (16, array([ 0.00058247,  0.00349755,  0.00204345, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (94, array([ 0.00077435,  0.00280832,  0.0015606 , ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (55, array([ 0.00061268,  0.00413733,  0.00203267, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (56, array([ 0.00045743,  0.0025789 ,  0.00160628, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (17, array([ 0.00059926,  0.00439214,  0.00150857, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (95, array([ 0.00061241,  0.00281041,  0.0022055 , ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (96, array([ 0.00035579,  0.00239502,  0.00142277, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (57, array([ 0.00063166,  0.00309013,  0.00202294, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (18, array([ 0.00067342,  0.00384218,  0.00230654, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (97, array([  5.79170365e-04,   3.04951088e-03,   1.91329565e-03, ...,\n",
       "            3.24946096e-05,   0.00000000e+00,   1.04324799e-04])),\n",
       "  (58, array([ 0.00058389,  0.00467036,  0.00212712, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (19, array([ 0.00057784,  0.00385314,  0.00189679, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (98, array([  6.54008884e-04,   3.10237162e-03,   2.22255072e-03, ...,\n",
       "            2.06181387e-05,   4.44820757e-05,   2.25409643e-05])),\n",
       "  (59, array([ 0.00062914,  0.00260365,  0.00166628, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (20, array([ 0.00028206,  0.00467405,  0.00238781, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (99, array([ 0.00057199,  0.00286962,  0.00178828, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (60, array([ 0.00052106,  0.00438765,  0.00174824, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (21, array([ 0.00045551,  0.00238862,  0.00095249, ...,  0.00958724,\n",
       "           0.        ,  0.        ])),\n",
       "  (61, array([ 0.00073372,  0.00324924,  0.00179214, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (22, array([ 0.00057106,  0.0035918 ,  0.00269398, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (62, array([ 0.00061352,  0.00256248,  0.00285221, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (23, array([ 0.00063886,  0.00348276,  0.00177374, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (24, array([ 0.00064676,  0.00307751,  0.00162758, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (63, array([ 0.00063131,  0.00258138,  0.00256058, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (64, array([ 0.0006128 ,  0.00281196,  0.00201214, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (25, array([  6.11401666e-04,   3.52808253e-03,   2.04977397e-03, ...,\n",
       "            6.62745773e-05,   0.00000000e+00,   0.00000000e+00])),\n",
       "  (65, array([ 0.00015153,  0.00664814,  0.00408914, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (26, array([  6.08733204e-04,   2.78830165e-03,   2.54211940e-03, ...,\n",
       "            6.13092640e-05,   7.10165641e-05,   0.00000000e+00])),\n",
       "  (66, array([ 0.0005944 ,  0.        ,  0.00119792, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (27, array([  5.50159544e-04,   4.14175213e-03,   2.05726166e-03, ...,\n",
       "            1.26540318e-05,   4.08510874e-05,   5.79934598e-06])),\n",
       "  (67, array([ 0.00055627,  0.00440454,  0.00179955, ...,  0.        ,\n",
       "           0.00013192,  0.        ])),\n",
       "  (28, array([ 0.00066691,  0.00386734,  0.00153336, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (68, array([  5.73569240e-04,   3.13389289e-03,   2.03593168e-03, ...,\n",
       "            0.00000000e+00,   7.36137132e-05,   0.00000000e+00])),\n",
       "  (29, array([ 0.00057907,  0.0032696 ,  0.00231529, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (69, array([  6.06732764e-04,   2.76933841e-03,   2.87326476e-03, ...,\n",
       "            1.18437255e-04,   1.40010751e-05,   0.00000000e+00])),\n",
       "  (30, array([ 0.00068787,  0.00309916,  0.00206417, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (70, array([ 0.00059731,  0.0039499 ,  0.00192296, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (31, array([ 0.00058822,  0.00343316,  0.00230508, ...,  0.00015482,\n",
       "           0.        ,  0.        ])),\n",
       "  (32, array([ 0.00035168,  0.01147899,  0.        , ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (71, array([  5.93578320e-04,   3.51386422e-03,   2.13178983e-03, ...,\n",
       "            2.90477180e-05,   0.00000000e+00,   1.41521277e-05])),\n",
       "  (72, array([ 0.00049462,  0.00078909,  0.00373792, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (33, array([ 0.00049802,  0.00403685,  0.00135236, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (73, array([  5.50239869e-04,   3.45612910e-03,   2.46312537e-03, ...,\n",
       "            0.00000000e+00,   6.49261614e-05,   0.00000000e+00])),\n",
       "  (34, array([ 0.00021477,  0.00233669,  0.00389553, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (74, array([  6.08434247e-04,   4.03203305e-03,   2.10058365e-03, ...,\n",
       "            8.13802939e-06,   2.90061684e-05,   1.25751921e-04])),\n",
       "  (35, array([ 0.00057115,  0.00214749,  0.00518172, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (75, array([ 0.00061297,  0.00291524,  0.00206339, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (36, array([ 0.00066481,  0.00428527,  0.00224157, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (76, array([  6.18432746e-04,   3.46690339e-03,   1.79954140e-03, ...,\n",
       "            2.23369781e-05,   0.00000000e+00,   0.00000000e+00])),\n",
       "  (37, array([ 0.00062114,  0.00135445,  0.00075268, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (77, array([ 0.00067498,  0.00268355,  0.00203194, ...,  0.        ,\n",
       "           0.        ,  0.        ])),\n",
       "  (38, array([ 0.00050241,  0.        ,  0.        , ...,  0.        ,\n",
       "           0.        ,  0.        ]))],\n",
       " [27967.33575621339, 22173.909203851395, 22022.730316505375])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_results = Kmeans(sample_data, centroids, 3)\n",
    "text_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   79, 13037],\n",
       "       [   98,  6957],\n",
       "       [   74,  6635],\n",
       "       [   27,  5514],\n",
       "       [   83,  3713],\n",
       "       [   64,  2545],\n",
       "       [   71,  2335],\n",
       "       [    8,  2174],\n",
       "       [   73,  1945],\n",
       "       [    9,  1862]])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_res = text_results[0].reduceByKey(lambda x, y:  (x[0] ,x[1] + y[1]))\n",
    "common = filtered_res.map(lambda tup: (tup[0], tup[1][1])).filter(lambda x: x[1] > 1).collect()\n",
    "topics = np.array(common)\n",
    "top_topics = np.argsort(topics[:,1])[::-1][:10]\n",
    "topics[top_topics]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top Words for each Cluster:\n",
      "\n",
      "79: '', world, math, We, would, work, community, They, Our, see, children, kids, learning, want, year, life, project, class, learn, make\n",
      "\n",
      "\n",
      "98: technology, iPad, computer, computers, iPads, access, use, laptop, research, apps, learning, classroom, would, math, world, using, '', skills, projects, able\n",
      "\n",
      "\n",
      "74: books, reading, read, library, book, readers, level, '', love, leveled, home, nonfiction, grade, interest, want, levels, classroom, children, new, non-fiction\n",
      "\n",
      "\n",
      "27: reading, '', read, book, books, words, readers, children, comprehension, grade, love, skills, Reading, would, fluency, level, materials, want, word, time\n",
      "\n",
      "\n",
      "83: language, Spanish, English, centers, math, materials, skills, learning, vocabulary, activities, concepts, fun, practice, words, children, '', hands-on, games, learners, learn\n",
      "\n",
      "\n",
      "64: physical, sensory, special, disabilities, motor, needs, education, play, skills, children, equipment, '', fine, activities, Autism, materials, fitness, healthy, speech, activity\n",
      "\n",
      "\n",
      "71: science, Science, hands-on, materials, '', scientists, lab, scientific, concepts, explore, experiments, hands, world, activities, kits, experience, learning, life, labs, learn\n",
      "\n",
      "\n",
      "8: carpet, rug, supplies, garden, sit, space, plants, place, floor, materials, classroom, pencils, '', children, year, basic, We, would, learning, day\n",
      "\n",
      "\n",
      "73: art, paint, supplies, paper, Art, projects, create, creative, express, painting, arts, materials, '', artists, creativity, drawing, color, artwork, markers, crayons\n",
      "\n",
      "\n",
      "9: math, manipulatives, concepts, fractions, hands-on, materials, Math, fraction, counting, numbers, skills, understanding, games, fun, hands, centers, activities, '', learning, understand\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "top_n = 20\n",
    "df = pd.DataFrame(text_results[1])\n",
    "\n",
    "df.columns = ['a', 'b']\n",
    "df_index = df.set_index('a')\n",
    "\n",
    "print \"Top Words for each Cluster:\\n\"\n",
    "\n",
    "for i , v in df_index.loc[topics[top_topics][:,0]].iterrows():\n",
    "    print \"%d: %s\" % (i, \", \".join(top_terms[j][0] for j in v.b.argsort()[::-1][:top_n]))\n",
    "    print \"\\n\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}