1.4.5 - Evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pyspark.context.SparkContext at 0x7f7ef14b8790>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.sql import SQLContext\n",
    "sqlc = SQLContext(sc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation Metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Binary Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2016-10-01 17:14:16--  https://raw.githubusercontent.com/apache/spark/master/data/mllib/sample_binary_classification_data.txt\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.12.133\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.12.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 104736 (102K) [text/plain]\n",
      "Saving to: ‘sample_binary_classification_data.txt.1’\n",
      "\n",
      "100%[======================================>] 104.736     --.-K/s   in 0,09s   \n",
      "\n",
      "2016-10-01 17:14:16 (1,09 MB/s) - ‘sample_binary_classification_data.txt.1’ saved [104736/104736]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.mllib.classification import LogisticRegressionWithLBFGS\n",
    "from pyspark.mllib.evaluation import BinaryClassificationMetrics\n",
    "from pyspark.mllib.regression import LabeledPoint\n",
    "from pyspark.mllib.util import MLUtils\n",
    "\n",
    "!wget https://raw.githubusercontent.com/apache/spark/master/data/mllib/sample_binary_classification_data.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = MLUtils.loadLibSVMFile(sc, \"sample_binary_classification_data.txt\")\n",
    "\n",
    "training, test = data.randomSplit([0.6, 0.4], seed=11L)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[LabeledPoint(0.0, (692,[127,128,129,130,131,154,155,156,157,158,159,181,182,183,184,185,186,187,188,189,207,208,209,210,211,212,213,214,215,216,217,235,236,237,238,239,240,241,242,243,244,245,262,263,264,265,266,267,268,269,270,271,272,273,289,290,291,292,293,294,295,296,297,300,301,302,316,317,318,319,320,321,328,329,330,343,344,345,346,347,348,349,356,357,358,371,372,373,374,384,385,386,399,400,401,412,413,414,426,427,428,429,440,441,442,454,455,456,457,466,467,468,469,470,482,483,484,493,494,495,496,497,510,511,512,520,521,522,523,538,539,540,547,548,549,550,566,567,568,569,570,571,572,573,574,575,576,577,578,594,595,596,597,598,599,600,601,602,603,604,622,623,624,625,626,627,628,629,630,651,652,653,654,655,656,657],[51.0,159.0,253.0,159.0,50.0,48.0,238.0,252.0,252.0,252.0,237.0,54.0,227.0,253.0,252.0,239.0,233.0,252.0,57.0,6.0,10.0,60.0,224.0,252.0,253.0,252.0,202.0,84.0,252.0,253.0,122.0,163.0,252.0,252.0,252.0,253.0,252.0,252.0,96.0,189.0,253.0,167.0,51.0,238.0,253.0,253.0,190.0,114.0,253.0,228.0,47.0,79.0,255.0,168.0,48.0,238.0,252.0,252.0,179.0,12.0,75.0,121.0,21.0,253.0,243.0,50.0,38.0,165.0,253.0,233.0,208.0,84.0,253.0,252.0,165.0,7.0,178.0,252.0,240.0,71.0,19.0,28.0,253.0,252.0,195.0,57.0,252.0,252.0,63.0,253.0,252.0,195.0,198.0,253.0,190.0,255.0,253.0,196.0,76.0,246.0,252.0,112.0,253.0,252.0,148.0,85.0,252.0,230.0,25.0,7.0,135.0,253.0,186.0,12.0,85.0,252.0,223.0,7.0,131.0,252.0,225.0,71.0,85.0,252.0,145.0,48.0,165.0,252.0,173.0,86.0,253.0,225.0,114.0,238.0,253.0,162.0,85.0,252.0,249.0,146.0,48.0,29.0,85.0,178.0,225.0,253.0,223.0,167.0,56.0,85.0,252.0,252.0,252.0,229.0,215.0,252.0,252.0,252.0,196.0,130.0,28.0,199.0,252.0,252.0,253.0,252.0,252.0,233.0,145.0,25.0,128.0,252.0,253.0,252.0,141.0,37.0]))]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.take(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Area under PR = 0.993386243386\n",
      "Area under ROC = 0.981481481481\n"
     ]
    }
   ],
   "source": [
    "!rm -rf metastore_db/*.lck\n",
    "\n",
    "model = LogisticRegressionWithLBFGS.train(training)\n",
    "\n",
    "predictionAndLabels = test.map(lambda lp: (float(model.predict(lp.features)), lp.label))\n",
    "\n",
    "metrics = BinaryClassificationMetrics(predictionAndLabels)\n",
    "\n",
    "print(\"Area under PR = %s\" % metrics.areaUnderPR)\n",
    "\n",
    "print(\"Area under ROC = %s\" % metrics.areaUnderROC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(1.0, 1.0)]\n"
     ]
    }
   ],
   "source": [
    "print predictionAndLabels.take(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multiclassification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2016-09-26 09:14:48--  https://raw.githubusercontent.com/apache/spark/master/data/mllib/sample_multiclass_classification_data.txt\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.12.133\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.12.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 6953 (6,8K) [text/plain]\n",
      "Saving to: ‘sample_multiclass_classification_data.txt’\n",
      "\n",
      "100%[======================================>] 6.953       --.-K/s   in 0s      \n",
      "\n",
      "2016-09-26 09:14:48 (27,5 MB/s) - ‘sample_multiclass_classification_data.txt’ saved [6953/6953]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.mllib.evaluation import MulticlassMetrics\n",
    "\n",
    "!wget https://raw.githubusercontent.com/apache/spark/master/data/mllib/sample_multiclass_classification_data.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[LabeledPoint(1.0, (4,[0,1,2,3],[-0.222222,0.5,-0.762712,-0.833333]))]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = MLUtils.loadLibSVMFile(sc, \"sample_multiclass_classification_data.txt\")\n",
    "\n",
    "training, test = data.randomSplit([0.6, 0.4], seed=11L)\n",
    "\n",
    "data.take(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = LogisticRegressionWithLBFGS.train(training, numClasses=3)\n",
    "\n",
    "predictionAndLabels = test.map(lambda lp: (float(model.predict(lp.features)), lp.label))\n",
    "\n",
    "metrics = MulticlassMetrics(predictionAndLabels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summary Stats\n",
      "Precision = 0.912280701754\n",
      "Recall = 0.912280701754\n",
      "F1 Score = 0.912280701754\n"
     ]
    }
   ],
   "source": [
    "precision = metrics.precision()\n",
    "recall = metrics.recall()\n",
    "f1Score = metrics.fMeasure()\n",
    "print(\"Summary Stats\")\n",
    "print(\"Precision = %s\" % precision)\n",
    "print(\"Recall = %s\" % recall)\n",
    "print(\"F1 Score = %s\" % f1Score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Class 0.0 precision = 0.913043478261\n",
      "Class 0.0 recall = 0.875\n",
      "Class 0.0 F1 Measure = 0.893617021277\n",
      "Class 1.0 precision = 1.0\n",
      "Class 1.0 recall = 1.0\n",
      "Class 1.0 F1 Measure = 1.0\n",
      "Class 2.0 precision = 0.8125\n",
      "Class 2.0 recall = 0.866666666667\n",
      "Class 2.0 F1 Measure = 0.838709677419\n"
     ]
    }
   ],
   "source": [
    "labels = data.map(lambda lp: lp.label).distinct().collect()\n",
    "for label in sorted(labels):\n",
    "    print(\"Class %s precision = %s\" % (label, metrics.precision(label)))\n",
    "    print(\"Class %s recall = %s\" % (label, metrics.recall(label)))\n",
    "    print(\"Class %s F1 Measure = %s\" % (label, metrics.fMeasure(label, beta=1.0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Weighted recall = 0.912280701754\n",
      "Weighted precision = 0.914044622426\n",
      "Weighted F(1) Score = 0.912762345122\n",
      "Weighted F(0.5) Score = 0.913437018999\n",
      "Weighted false positive rate = 0.044315333789\n"
     ]
    }
   ],
   "source": [
    "print(\"Weighted recall = %s\" % metrics.weightedRecall)\n",
    "print(\"Weighted precision = %s\" % metrics.weightedPrecision)\n",
    "print(\"Weighted F(1) Score = %s\" % metrics.weightedFMeasure())\n",
    "print(\"Weighted F(0.5) Score = %s\" % metrics.weightedFMeasure(beta=0.5))\n",
    "print(\"Weighted false positive rate = %s\" % metrics.weightedFalsePositiveRate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2016-09-26 09:18:52--  https://raw.githubusercontent.com/apache/spark/master/data/mllib/sample_linear_regression_data.txt\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.12.133\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.12.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 119069 (116K) [text/plain]\n",
      "Saving to: ‘sample_linear_regression_data.txt.1’\n",
      "\n",
      "100%[======================================>] 119.069     --.-K/s   in 0,09s   \n",
      "\n",
      "2016-09-26 09:18:53 (1,23 MB/s) - ‘sample_linear_regression_data.txt.1’ saved [119069/119069]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.mllib.regression import LinearRegressionWithSGD\n",
    "from pyspark.mllib.evaluation import RegressionMetrics\n",
    "from pyspark.mllib.linalg import DenseVector\n",
    "\n",
    "!wget https://raw.githubusercontent.com/apache/spark/master/data/mllib/sample_linear_regression_data.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[LabeledPoint(-9.49000987882, [0.455127360066,0.36644694352,-0.382561089335,-0.445843019852,0.331097903589,0.806744529344,-0.262434173177,-0.448503861117,-0.0726928483817,0.56580355758])]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def parsePoint(line):\n",
    "    values = line.split()\n",
    "    return LabeledPoint(float(values[0]), DenseVector([float(x.split(':')[1]) for x in values[1:]]))\n",
    "\n",
    "data = sc.textFile(\"sample_linear_regression_data.txt\")\n",
    "parsedData = data.map(parsePoint)\n",
    "\n",
    "parsedData.take(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = LinearRegressionWithSGD.train(parsedData)\n",
    "\n",
    "valuesAndPreds = parsedData.map(lambda p: (float(model.predict(p.features)), p.label))\n",
    "\n",
    "metrics = RegressionMetrics(valuesAndPreds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE = 103.309686818\n",
      "RMSE = 10.1641372884\n",
      "R-squared = 0.0276391109678\n",
      "MAE = 8.14869190795\n",
      "Explained variance = 2.88839520172\n"
     ]
    }
   ],
   "source": [
    "print(\"MSE = %s\" % metrics.meanSquaredError)\n",
    "print(\"RMSE = %s\" % metrics.rootMeanSquaredError)\n",
    "\n",
    "print(\"R-squared = %s\" % metrics.r2)\n",
    "\n",
    "print(\"MAE = %s\" % metrics.meanAbsoluteError)\n",
    "\n",
    "print(\"Explained variance = %s\" % metrics.explainedVariance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}