DF 3.4.5 - Evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pyspark.context.SparkContext at 0x7fa288754790>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!rm -rf metastore_db/*.lck\n",
    "\n",
    "from pyspark.sql import SQLContext\n",
    "sqlc = SQLContext(sc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from pyspark.mllib.util import MLUtils\n",
    "\n",
    "data = MLUtils.loadLibSVMFile(sc, \"sample_libsvm_data.txt\").toDF()\n",
    "data = MLUtils.convertVectorColumnsToML(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+-----+\n",
      "|            features|label|\n",
      "+--------------------+-----+\n",
      "|(692,[127,128,129...|  0.0|\n",
      "|(692,[158,159,160...|  1.0|\n",
      "|(692,[124,125,126...|  1.0|\n",
      "|(692,[152,153,154...|  1.0|\n",
      "|(692,[151,152,153...|  1.0|\n",
      "+--------------------+-----+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "data.show(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Binary Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Weights: (692,[235,243,244,262,263,271,272,290,300,301,323,328,329,350,351,358,378,379,385,386,401,405,406,407,413,427,428,429,433,434,435,455,456,461,462,469,483,484,489,490,496,497,511,512,517,539,540,568],[-1.08704076463e-05,-0.000225035191295,-1.42785721134e-06,-2.67838954428e-06,-0.000260749167911,-0.000108151886583,-0.000222047179992,-0.000184684690252,-0.000368205319861,-8.45317828463e-05,8.41700764597e-05,-8.99150674055e-06,-2.6018144065e-06,0.000106542521448,0.000214847611674,-8.02001292779e-06,0.000254341232325,0.000218251840905,-3.96099160439e-05,-2.89412079021e-05,-4.45357563079e-06,5.02740058996e-05,0.00084364056682,0.000260914756075,-2.13472989153e-05,-1.2101806874e-06,-0.000135806709886,-9.35571716832e-06,0.000231569631081,0.000943064886038,7.40416456292e-05,-1.3984003133e-05,-0.000254944102754,0.000131412325807,0.000884111338263,-9.88044455622e-05,-8.10057764206e-05,-0.000193685045759,0.00011911644026,0.000142456462059,-0.000123316509266,-8.01517008313e-05,-0.000390099390576,-0.000394491923359,0.00016643986746,-0.000267852255111,-0.000848200730725,-0.000151726889697]) Intercept: 0.297623966973\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.classification import LogisticRegression\n",
    "from pyspark.ml.classification import BinaryLogisticRegressionSummary\n",
    "\n",
    "trainingData, testData = data.randomSplit([0.7, 0.3])\n",
    "\n",
    "logr = LogisticRegression().setMaxIter(10).setRegParam(0.3).setElasticNetParam(0.8)\n",
    "\n",
    "logrModel = logr.fit(trainingData)\n",
    "\n",
    "print \"Weights: %s Intercept: %s\" % (logrModel.coefficients, logrModel.intercept)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.evaluation import BinaryClassificationEvaluator\n",
    "\n",
    "predictionsLogR = logrModel.transform(testData)\n",
    "\n",
    "evaluator = BinaryClassificationEvaluator().setLabelCol(\"label\") \\\n",
    "                            .setRawPredictionCol(\"rawPrediction\") \\\n",
    "                            .setMetricName(\"areaUnderROC\")\n",
    "\n",
    "roc = evaluator.evaluate(predictionsLogR)\n",
    "print roc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multiclass Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.feature import StringIndexer, IndexToString, VectorIndexer\n",
    "from pyspark.ml.classification import RandomForestClassifier\n",
    "from pyspark.ml.classification import RandomForestClassificationModel\n",
    "\n",
    "labelIndexer = StringIndexer().setInputCol(\"label\") \\\n",
    "                                .setOutputCol(\"indexedLabel\").fit(data)\n",
    "\n",
    "labelConverter = IndexToString().setInputCol(\"prediction\") \\\n",
    "                                .setOutputCol(\"predictedLabel\") \\\n",
    "                                .setLabels(labelIndexer.labels)\n",
    "\n",
    "featureIndexer = VectorIndexer().setInputCol(\"features\") \\\n",
    "                                .setOutputCol(\"indexedFeatures\") \\\n",
    "                                .setMaxCategories(4).fit(data)\n",
    "\n",
    "rfC = RandomForestClassifier().setLabelCol(\"indexedLabel\") \\\n",
    "                                .setFeaturesCol(\"indexedFeatures\") \\\n",
    "                                .setNumTrees(3)\n",
    "        \n",
    "pipelineRFC = Pipeline().setStages([labelIndexer, featureIndexer, rfC, labelConverter])\n",
    "\n",
    "modelRFC = pipelineRFC.fit(trainingData)\n",
    "\n",
    "predictionsRFC = modelRFC.transform(testData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Error = 0.0606060606061\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
    "\n",
    "evaluator = MulticlassClassificationEvaluator().setLabelCol(\"indexedLabel\") \\\n",
    "                                        .setPredictionCol(\"prediction\") \\\n",
    "                                        .setMetricName(\"accuracy\")\n",
    "\n",
    "accuracy = evaluator.evaluate(predictionsRFC)\n",
    "\n",
    "print \"Test Error = %s\" % (1.0 - accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.ml.regression import RandomForestRegressor\n",
    "from pyspark.ml.regression import RandomForestRegressionModel\n",
    "\n",
    "rfR = RandomForestRegressor().setLabelCol(\"label\").setFeaturesCol(\"indexedFeatures\")\n",
    "\n",
    "pipelineRFR = Pipeline().setStages([featureIndexer, rfR])\n",
    "\n",
    "modelRFR = pipelineRFR.fit(trainingData)\n",
    "\n",
    "predictionsRFR = modelRFR.transform(testData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Root Mean Squared Error (RMSE) = 0.138169855942\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.evaluation import RegressionEvaluator\n",
    "\n",
    "evaluator = RegressionEvaluator().setLabelCol(\"label\") \\\n",
    "                                .setPredictionCol(\"prediction\") \\\n",
    "                                .setMetricName(\"rmse\")\n",
    "\n",
    "rmse = evaluator.evaluate(predictionsRFR)\n",
    "\n",
    "print \"Root Mean Squared Error (RMSE) = %s\" % rmse"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Weights: (692,[235,243,244,262,263,271,272,290,300,301,323,328,329,350,351,358,378,379,385,386,401,405,406,407,413,427,428,429,433,434,435,455,456,461,462,469,483,484,489,490,496,497,511,512,517,539,540,568],[-1.08704076463e-05,-0.000225035191295,-1.42785721134e-06,-2.67838954428e-06,-0.000260749167911,-0.000108151886583,-0.000222047179992,-0.000184684690252,-0.000368205319861,-8.45317828463e-05,8.41700764597e-05,-8.99150674055e-06,-2.6018144065e-06,0.000106542521448,0.000214847611674,-8.02001292779e-06,0.000254341232325,0.000218251840905,-3.96099160439e-05,-2.89412079021e-05,-4.45357563079e-06,5.02740058996e-05,0.00084364056682,0.000260914756075,-2.13472989153e-05,-1.2101806874e-06,-0.000135806709886,-9.35571716832e-06,0.000231569631081,0.000943064886038,7.40416456292e-05,-1.3984003133e-05,-0.000254944102754,0.000131412325807,0.000884111338263,-9.88044455622e-05,-8.10057764206e-05,-0.000193685045759,0.00011911644026,0.000142456462059,-0.000123316509266,-8.01517008313e-05,-0.000390099390576,-0.000394491923359,0.00016643986746,-0.000267852255111,-0.000848200730725,-0.000151726889697]) Intercept: 0.297623966973\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.classification import LogisticRegression\n",
    "\n",
    "logr = LogisticRegression().setMaxIter(10).setRegParam(0.3).setElasticNetParam(0.8)\n",
    "\n",
    "logrModel = logr.fit(trainingData)\n",
    "\n",
    "print \"Weights: %s Intercept: %s\" % (logrModel.coefficients, logrModel.intercept)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainingSummaryLR = logrModel.summary\n",
    "trainingSummaryLR.areaUnderROC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------+-------------------+\n",
      "|         threshold|          F-Measure|\n",
      "+------------------+-------------------+\n",
      "|0.8165428632958064|               0.05|\n",
      "|0.8162694676199894|0.09756097560975609|\n",
      "|0.8162646945004736|0.14285714285714288|\n",
      "+------------------+-------------------+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "fMeasure = trainingSummaryLR.fMeasureByThreshold\n",
    "\n",
    "fMeasure.show(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0\n",
      "1.0\n",
      "0.613903946897\n"
     ]
    }
   ],
   "source": [
    "from pyspark.sql import functions as F\n",
    "\n",
    "maxFMeasure = fMeasure.agg({\"F-Measure\": \"max\"}).head()[0]\n",
    "print maxFMeasure\n",
    "maxFMeasure = fMeasure.agg(F.max(F.col(\"F-Measure\"))).head()[0]\n",
    "print maxFMeasure\n",
    "\n",
    "bestThreshold = fMeasure.where(F.col(\"F-Measure\") == maxFMeasure).select(\"threshold\").head()[0]\n",
    "print bestThreshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------+---------+\n",
      "|             recall|precision|\n",
      "+-------------------+---------+\n",
      "|                0.0|      1.0|\n",
      "|0.02564102564102564|      1.0|\n",
      "|0.05128205128205128|      1.0|\n",
      "+-------------------+---------+\n",
      "only showing top 3 rows\n",
      "\n",
      "+------------------+---------+\n",
      "|         threshold|precision|\n",
      "+------------------+---------+\n",
      "|0.8165428632958064|      1.0|\n",
      "|0.8162694676199894|      1.0|\n",
      "|0.8162646945004736|      1.0|\n",
      "+------------------+---------+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "trainingSummaryLR.pr.show(3)\n",
    "trainingSummaryLR.precisionByThreshold.show(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------+-------------------+\n",
      "|         threshold|             recall|\n",
      "+------------------+-------------------+\n",
      "|0.8165428632958064|0.02564102564102564|\n",
      "|0.8162694676199894|0.05128205128205128|\n",
      "|0.8162646945004736|0.07692307692307693|\n",
      "+------------------+-------------------+\n",
      "only showing top 3 rows\n",
      "\n",
      "+---+-------------------+\n",
      "|FPR|                TPR|\n",
      "+---+-------------------+\n",
      "|0.0|                0.0|\n",
      "|0.0|0.02564102564102564|\n",
      "|0.0|0.05128205128205128|\n",
      "+---+-------------------+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "trainingSummaryLR.recallByThreshold.show(3)\n",
    "trainingSummaryLR.roc.show(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Linear Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Weights: (692,[351,378,379,406,407,433,434,462,490,517,540],[0.000133767304687,0.00018419522405,0.000135554811074,0.000275780406915,0.000159724075215,0.000144485199988,0.000306915977164,0.000288535540884,0.000120301061362,0.000123576266478,-0.00011823301358]) Intercept: 0.34681682126\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.regression import LinearRegression\n",
    "\n",
    "lr = LinearRegression().setMaxIter(10).setRegParam(0.3).setElasticNetParam(0.8)\n",
    "\n",
    "lrModel = lr.fit(trainingData)\n",
    "\n",
    "print \"Weights: %s Intercept: %s\" % (lrModel.coefficients, lrModel.intercept)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0506615309323\n",
      "0.271181060072\n",
      "0.0785812855913\n",
      "0.676967590642\n"
     ]
    }
   ],
   "source": [
    "trainingSummaryLLS = lrModel.summary\n",
    "\n",
    "print trainingSummaryLLS.explainedVariance\n",
    "\n",
    "print trainingSummaryLLS.meanAbsoluteError\n",
    "\n",
    "print trainingSummaryLLS.meanSquaredError\n",
    "\n",
    "print trainingSummaryLLS.r2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------+\n",
      "|          residuals|\n",
      "+-------------------+\n",
      "|0.24739337680570395|\n",
      "|-0.3169038688245795|\n",
      "| 0.5361913707012146|\n",
      "+-------------------+\n",
      "only showing top 3 rows\n",
      "\n",
      "0.280323537348\n"
     ]
    }
   ],
   "source": [
    "trainingSummaryLLS.residuals.show(3)\n",
    "\n",
    "print trainingSummaryLLS.rootMeanSquaredError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}