4. MLib.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MLib is Machine Learning Library for Spark\n",
    "1. Incorporates with Numpy in python\n",
    "2. It provied an Integrated data analysis Flow\n",
    "3. Enhances speed and performances\n",
    "4. Clustering, Feature Pattern Matching, Linear Algebra, Collaborative filtering, classification, regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pyspark\n",
    "from pyspark.sql import SparkSession\n",
    "sc = pyspark.SparkContext(appName=\"mlib\")\n",
    "spark = SparkSession.builder.appName('mlib').getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+---+---+--------+-----+---+-------+-------+-----+-------+-----+---+----+---+\n",
      "| age|sex| cp|trestbps| chol|fbs|restecg|thalach|exang|oldpeak|slope| ca|thal|num|\n",
      "+----+---+---+--------+-----+---+-------+-------+-----+-------+-----+---+----+---+\n",
      "|67.0|1.0|4.0|   160.0|286.0|0.0|    2.0|  108.0|  1.0|    1.5|  2.0|3.0| 3.0|  2|\n",
      "|67.0|1.0|4.0|   120.0|229.0|0.0|    2.0|  129.0|  1.0|    2.6|  2.0|2.0| 7.0|  1|\n",
      "|37.0|1.0|3.0|   130.0|250.0|0.0|    0.0|  187.0|  0.0|    3.5|  3.0|0.0| 3.0|  0|\n",
      "|41.0|0.0|2.0|   130.0|204.0|0.0|    2.0|  172.0|  0.0|    1.4|  1.0|0.0| 3.0|  0|\n",
      "|56.0|1.0|2.0|   120.0|236.0|0.0|    0.0|  178.0|  0.0|    0.8|  1.0|0.0| 3.0|  0|\n",
      "+----+---+---+--------+-----+---+-------+-------+-----+-------+-----+---+----+---+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = spark.read.csv(\"file:///home/aditya/GitHub Projects/Spark-Tutorials/Resources/heart.csv\",inferSchema=True,header=True)\n",
    "df.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "df_pd = df.toPandas()\n",
    "df_pd.replace('?',np.NaN,inplace=True)\n",
    "df_pd.dropna(inplace=True)\n",
    "df_pd.to_csv(\"Resources/heart_new.csv\",header=False,index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.mllib.regression import LabeledPoint\n",
    "points = sc.textFile(\"file:///home/aditya/GitHub Projects/Spark-Tutorials/Resources/heart_new.csv\")\n",
    "def prepareLabelledPoints(row):\n",
    "    values = [float(s) for s in row.strip().split(',')]\n",
    "    return LabeledPoint(int(values[13]),values[:13])\n",
    "pointrdd = points.map(prepareLabelledPoints)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[LabeledPoint(2.0, [67.0,1.0,4.0,160.0,286.0,0.0,2.0,108.0,1.0,1.5,2.0,3.0,3.0]),\n",
       " LabeledPoint(1.0, [67.0,1.0,4.0,120.0,229.0,0.0,2.0,129.0,1.0,2.6,2.0,2.0,7.0]),\n",
       " LabeledPoint(0.0, [37.0,1.0,3.0,130.0,250.0,0.0,0.0,187.0,0.0,3.5,3.0,0.0,3.0]),\n",
       " LabeledPoint(0.0, [41.0,0.0,2.0,130.0,204.0,0.0,2.0,172.0,0.0,1.4,1.0,0.0,3.0]),\n",
       " LabeledPoint(0.0, [56.0,1.0,2.0,120.0,236.0,0.0,0.0,178.0,0.0,0.8,1.0,0.0,3.0])]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pointrdd.take(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.mllib.tree import DecisionTree\n",
    "(trainData,testData) = pointrdd.randomSplit([0.7,0.3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trained_model = DecisionTree.trainClassifier(trainData,numClasses=5,categoricalFeaturesInfo={},impurity='gini',maxDepth=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.0, 2.0, 2.0, 0.0, 0.0, 0.0, 0.0, 1.0, 2.0, 3.0]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions = trained_model.predict(testData.map(lambda x: x.features))\n",
    "predictions.take(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "labelsandpredictions = testData.map(lambda lp: lp.label).zip(predictions)\n",
    "labelPrediction=labelsandpredictions.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy :0.5555555555555556\n"
     ]
    }
   ],
   "source": [
    "total = len(labelPrediction)\n",
    "correct = len(list(filter(lambda x: x[0]==x[1],labelPrediction)))\n",
    "print(\"Accuracy :{}\".format(correct/total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}