LowHumidityPrediction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.sql import SQLContext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sqlContext = SQLContext(sc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = sqlContext.read.load('file:///home/cloudera/Downloads/big-data-4/daily_weather.csv',\n",
    "                         format='com.databricks.spark.csv',\n",
    "                         header='true', inferSchema='true')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['number',\n",
       " 'air_pressure_9am',\n",
       " 'air_temp_9am',\n",
       " 'avg_wind_direction_9am',\n",
       " 'avg_wind_speed_9am',\n",
       " 'max_wind_direction_9am',\n",
       " 'max_wind_speed_9am',\n",
       " 'rain_accumulation_9am',\n",
       " 'rain_duration_9am',\n",
       " 'relative_humidity_9am',\n",
       " 'relative_humidity_3pm']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- number: integer (nullable = true)\n",
      " |-- air_pressure_9am: double (nullable = true)\n",
      " |-- air_temp_9am: double (nullable = true)\n",
      " |-- avg_wind_direction_9am: double (nullable = true)\n",
      " |-- avg_wind_speed_9am: double (nullable = true)\n",
      " |-- max_wind_direction_9am: double (nullable = true)\n",
      " |-- max_wind_speed_9am: double (nullable = true)\n",
      " |-- rain_accumulation_9am: double (nullable = true)\n",
      " |-- rain_duration_9am: double (nullable = true)\n",
      " |-- relative_humidity_9am: double (nullable = true)\n",
      " |-- relative_humidity_3pm: double (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>summary</th>\n",
       "      <td>count</td>\n",
       "      <td>mean</td>\n",
       "      <td>stddev</td>\n",
       "      <td>min</td>\n",
       "      <td>max</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>number</th>\n",
       "      <td>1095</td>\n",
       "      <td>547.0</td>\n",
       "      <td>316.24357700987383</td>\n",
       "      <td>0</td>\n",
       "      <td>1094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>air_pressure_9am</th>\n",
       "      <td>1092</td>\n",
       "      <td>918.8825513138097</td>\n",
       "      <td>3.1841611803868353</td>\n",
       "      <td>907.9900000000024</td>\n",
       "      <td>929.3200000000012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>air_temp_9am</th>\n",
       "      <td>1090</td>\n",
       "      <td>64.93300141287075</td>\n",
       "      <td>11.175514003175877</td>\n",
       "      <td>36.752000000000685</td>\n",
       "      <td>98.90599999999992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>avg_wind_direction_9am</th>\n",
       "      <td>1091</td>\n",
       "      <td>142.23551070057584</td>\n",
       "      <td>69.13785928889183</td>\n",
       "      <td>15.500000000000046</td>\n",
       "      <td>343.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>avg_wind_speed_9am</th>\n",
       "      <td>1092</td>\n",
       "      <td>5.50828424225493</td>\n",
       "      <td>4.552813465531715</td>\n",
       "      <td>0.69345139999974</td>\n",
       "      <td>23.554978199999763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max_wind_direction_9am</th>\n",
       "      <td>1092</td>\n",
       "      <td>148.9535179651692</td>\n",
       "      <td>67.23801294602951</td>\n",
       "      <td>28.89999999999991</td>\n",
       "      <td>312.19999999999993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max_wind_speed_9am</th>\n",
       "      <td>1091</td>\n",
       "      <td>7.019513529175272</td>\n",
       "      <td>5.59820917078096</td>\n",
       "      <td>1.1855782000000479</td>\n",
       "      <td>29.84077959999996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rain_accumulation_9am</th>\n",
       "      <td>1089</td>\n",
       "      <td>0.20307895225211126</td>\n",
       "      <td>1.5939521253574904</td>\n",
       "      <td>0.0</td>\n",
       "      <td>24.01999999999907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rain_duration_9am</th>\n",
       "      <td>1092</td>\n",
       "      <td>294.1080522756142</td>\n",
       "      <td>1598.078778660148</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17704.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>relative_humidity_9am</th>\n",
       "      <td>1095</td>\n",
       "      <td>34.24140205923539</td>\n",
       "      <td>25.472066802250044</td>\n",
       "      <td>6.090000000001012</td>\n",
       "      <td>92.6200000000002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>relative_humidity_3pm</th>\n",
       "      <td>1095</td>\n",
       "      <td>35.34472714825902</td>\n",
       "      <td>22.52407945358728</td>\n",
       "      <td>5.3000000000006855</td>\n",
       "      <td>92.2500000000003</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            0                    1                   2  \\\n",
       "summary                 count                 mean              stddev   \n",
       "number                   1095                547.0  316.24357700987383   \n",
       "air_pressure_9am         1092    918.8825513138097  3.1841611803868353   \n",
       "air_temp_9am             1090    64.93300141287075  11.175514003175877   \n",
       "avg_wind_direction_9am   1091   142.23551070057584   69.13785928889183   \n",
       "avg_wind_speed_9am       1092     5.50828424225493   4.552813465531715   \n",
       "max_wind_direction_9am   1092    148.9535179651692   67.23801294602951   \n",
       "max_wind_speed_9am       1091    7.019513529175272    5.59820917078096   \n",
       "rain_accumulation_9am    1089  0.20307895225211126  1.5939521253574904   \n",
       "rain_duration_9am        1092    294.1080522756142   1598.078778660148   \n",
       "relative_humidity_9am    1095    34.24140205923539  25.472066802250044   \n",
       "relative_humidity_3pm    1095    35.34472714825902   22.52407945358728   \n",
       "\n",
       "                                         3                   4  \n",
       "summary                                min                 max  \n",
       "number                                   0                1094  \n",
       "air_pressure_9am         907.9900000000024   929.3200000000012  \n",
       "air_temp_9am            36.752000000000685   98.90599999999992  \n",
       "avg_wind_direction_9am  15.500000000000046               343.4  \n",
       "avg_wind_speed_9am        0.69345139999974  23.554978199999763  \n",
       "max_wind_direction_9am   28.89999999999991  312.19999999999993  \n",
       "max_wind_speed_9am      1.1855782000000479   29.84077959999996  \n",
       "rain_accumulation_9am                  0.0   24.01999999999907  \n",
       "rain_duration_9am                      0.0             17704.0  \n",
       "relative_humidity_9am    6.090000000001012    92.6200000000002  \n",
       "relative_humidity_3pm   5.3000000000006855    92.2500000000003  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check summary statistics\n",
    "df.describe().toPandas().transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+\n",
      "|summary|  air_pressure_9am|\n",
      "+-------+------------------+\n",
      "|  count|              1092|\n",
      "|   mean| 918.8825513138097|\n",
      "| stddev|3.1841611803868353|\n",
      "|    min| 907.9900000000024|\n",
      "|    max| 929.3200000000012|\n",
      "+-------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.describe('air_pressure_9am').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1095"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    " df2 = df.na.drop(subset=['air_pressure_9am'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1092"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7298253479609015"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.stat.corr(\"rain_accumulation_9am\", \"rain_duration_9am\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1089"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.na.drop(subset=['rain_accumulation_9am']).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8823388530772411"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " df2.stat.corr(\"relative_humidity_9am\", \"relative_humidity_3pm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+\n",
      "|summary|      air_temp_9am|\n",
      "+-------+------------------+\n",
      "|  count|              1090|\n",
      "|   mean| 64.93300141287075|\n",
      "| stddev|11.175514003175877|\n",
      "|    min|36.752000000000685|\n",
      "|    max| 98.90599999999992|\n",
      "+-------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Start Data Preparation assignment\n",
    "df.describe(['air_temp_9am']).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1095"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Drop all the rows with missing a value\n",
    "removeAllDF = df.na.drop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+\n",
      "|summary|      air_temp_9am|\n",
      "+-------+------------------+\n",
      "|  count|              1064|\n",
      "|   mean| 65.02260949558739|\n",
      "| stddev|11.168033449415699|\n",
      "|    min|36.752000000000685|\n",
      "|    max| 98.90599999999992|\n",
      "+-------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "removeAllDF.describe(['air_temp_9am']).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1064"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "removeAllDF.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Impute missing values with mean value for that column\n",
    "from pyspark.sql.functions import avg\n",
    "imputeDF = df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number 545.0018796992481\n",
      "air_pressure_9am 918.9031798641055\n",
      "air_temp_9am 65.02260949558739\n",
      "avg_wind_direction_9am 142.30675564934032\n",
      "avg_wind_speed_9am 5.485793050713691\n",
      "max_wind_direction_9am 148.48042413321312\n",
      "max_wind_speed_9am 6.9997136588756925\n",
      "rain_accumulation_9am 0.18202347650615522\n",
      "rain_duration_9am 266.3936973996038\n",
      "relative_humidity_9am 34.07743985327712\n",
      "relative_humidity_3pm 35.14838093290537\n"
     ]
    }
   ],
   "source": [
    "for x in imputeDF.columns:\n",
    "    meanValue = removeAllDF.agg(avg(x)).first()[0]\n",
    "    print(x, meanValue)\n",
    "    imputeDF = imputeDF.na.fill(meanValue, [x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+\n",
      "|summary|      air_temp_9am|\n",
      "+-------+------------------+\n",
      "|  count|              1090|\n",
      "|   mean| 64.93300141287075|\n",
      "| stddev|11.175514003175877|\n",
      "|    min|36.752000000000685|\n",
      "|    max| 98.90599999999992|\n",
      "+-------+------------------+\n",
      "\n",
      "+-------+------------------+\n",
      "|summary|      air_temp_9am|\n",
      "+-------+------------------+\n",
      "|  count|              1095|\n",
      "|   mean| 64.93341058219822|\n",
      "| stddev|11.149948199920226|\n",
      "|    min|36.752000000000685|\n",
      "|    max| 98.90599999999992|\n",
      "+-------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.describe(['air_temp_9am']).show()\n",
    "imputeDF.describe(['air_temp_9am']).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Classification work starts here\n",
    "from pyspark.sql import SQLContext\n",
    "from pyspark.sql import DataFrameNaFunctions\n",
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.classification import DecisionTreeClassifier\n",
    "from pyspark.ml.feature import Binarizer\n",
    "from pyspark.ml.feature import VectorAssembler, StringIndexer, VectorIndexer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "featureColumns = ['air_pressure_9am','air_temp_9am','avg_wind_direction_9am','avg_wind_speed_9am',\n",
    "        'max_wind_direction_9am','max_wind_speed_9am','rain_accumulation_9am',\n",
    "        'rain_duration_9am']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "droppedDF = df.drop('number')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['air_pressure_9am',\n",
       " 'air_temp_9am',\n",
       " 'avg_wind_direction_9am',\n",
       " 'avg_wind_speed_9am',\n",
       " 'max_wind_direction_9am',\n",
       " 'max_wind_speed_9am',\n",
       " 'rain_accumulation_9am',\n",
       " 'rain_duration_9am',\n",
       " 'relative_humidity_9am',\n",
       " 'relative_humidity_3pm']"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "droppedDF.columns\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "imputeDropDF = droppedDF.na.drop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1095, 11)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.count(), len(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1064, 10)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imputeDropDF.count(), len(imputeDropDF.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "binarizer = Binarizer(threshold=24.99999, inputCol=\"relative_humidity_3pm\", outputCol=\"label\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "binarizedDF = binarizer.transform(imputeDropDF)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------------+-----+\n",
      "|relative_humidity_3pm|label|\n",
      "+---------------------+-----+\n",
      "|   36.160000000000494|  1.0|\n",
      "|     19.4265967985621|  0.0|\n",
      "|   14.460000000000045|  0.0|\n",
      "|   12.742547353761848|  0.0|\n",
      "+---------------------+-----+\n",
      "only showing top 4 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "binarizedDF.select(\"relative_humidity_3pm\", \"label\").show(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "assembler = VectorAssembler(inputCols=featureColumns, outputCol=\"features\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "assembledDF = assembler.transform(binarizedDF)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "(trainingDataDF, testDataDF) = assembledDF.randomSplit([0.8,0.2], seed=13234)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(854, 210)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainingDataDF.count(), testDataDF.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dt = DecisionTreeClassifier(labelCol=\"label\", featuresCol=\"features\", maxDepth=5, minInstancesPerNode = 20, impurity=\"gini\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pipeline = Pipeline(stages=[dt])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = pipeline.fit(trainingDataDF)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions = model.transform(testDataDF)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+-----+\n",
      "|prediction|label|\n",
      "+----------+-----+\n",
      "|       1.0|  1.0|\n",
      "|       1.0|  1.0|\n",
      "|       1.0|  1.0|\n",
      "|       1.0|  1.0|\n",
      "|       1.0|  1.0|\n",
      "|       1.0|  1.0|\n",
      "|       0.0|  0.0|\n",
      "|       1.0|  1.0|\n",
      "|       1.0|  1.0|\n",
      "|       1.0|  1.0|\n",
      "|       0.0|  0.0|\n",
      "|       1.0|  1.0|\n",
      "|       1.0|  1.0|\n",
      "|       1.0|  1.0|\n",
      "|       0.0|  0.0|\n",
      "|       1.0|  1.0|\n",
      "|       1.0|  1.0|\n",
      "|       0.0|  0.0|\n",
      "|       0.0|  1.0|\n",
      "|       1.0|  1.0|\n",
      "+----------+-----+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.select(\"prediction\", \"label\").show(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#save prediction to CSV\n",
    "predictions.select(\"prediction\", \"label\").write.save(path='file:///home/cloudera/Downloads/big-data-4/predictions1.csv',\n",
    "                                                    format='com.databricks.spark.csv',\n",
    "                                                    header='true')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from pyspark.sql import SQLContext\n",
    "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
    "from pyspark.mllib.evaluation import MulticlassMetrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+-----+\n",
      "|prediction|label|\n",
      "+----------+-----+\n",
      "|       1.0|  1.0|\n",
      "|       1.0|  1.0|\n",
      "|       1.0|  1.0|\n",
      "|       1.0|  1.0|\n",
      "|       1.0|  1.0|\n",
      "|       1.0|  1.0|\n",
      "|       0.0|  0.0|\n",
      "|       1.0|  1.0|\n",
      "|       1.0|  1.0|\n",
      "|       1.0|  1.0|\n",
      "|       0.0|  0.0|\n",
      "|       1.0|  1.0|\n",
      "|       1.0|  1.0|\n",
      "|       1.0|  1.0|\n",
      "|       0.0|  0.0|\n",
      "|       1.0|  1.0|\n",
      "|       1.0|  1.0|\n",
      "|       0.0|  0.0|\n",
      "|       0.0|  1.0|\n",
      "|       1.0|  1.0|\n",
      "+----------+-----+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.select(\"prediction\", \"label\").show(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "evaluator = MulticlassClassificationEvaluator (\n",
    "labelCol = \"label\", predictionCol=\"prediction\", metricName=\"precision\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "accuracy = evaluator.evaluate(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.809524\n",
      "Accuracy percentage = 81\n",
      "Accuracy percentage1 =                                                                                             0.809524\n",
      "Accuracy percentage2 =                                                                                                 0.81\n",
      "Error = 0.190476\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy = %g\" % (accuracy))\n",
    "print(\"Accuracy percentage = %.2g\" % (accuracy * 100))\n",
    "print(\"Accuracy percentage1 = %100g\" % (accuracy))\n",
    "print(\"Accuracy percentage2 = %100.2g\" % (accuracy))\n",
    "print(\"Error = %g\" % (1- accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(prediction=1.0, label=1.0), Row(prediction=1.0, label=1.0)]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions.select(\"prediction\", \"label\").rdd.take(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1.0, 1.0), (1.0, 1.0)]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions.select(\"prediction\", \"label\").rdd.map(tuple).take(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "metrics = MulticlassMetrics(predictions.select(\"prediction\", \"label\").rdd.map(tuple))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 87.,  26.],\n",
       "       [ 14.,  83.]])"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.confusionMatrix().toArray().transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 87.,  14.],\n",
       "       [ 26.,  83.]])"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.confusionMatrix().toArray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}