From d9a09ef2ee598f90134e7e3d59128d28ddbb1dc0 Mon Sep 17 00:00:00 2001 From: Andrew Bullen Date: Tue, 4 Nov 2014 13:28:52 -0800 Subject: [PATCH 1/5] Make Binary Evaluation Metrics functions defined in cases where there are 0 positive or 0 negative examples. --- .../BinaryClassificationMetricComputers.scala | 24 ++++++-- .../BinaryClassificationMetricsSuite.scala | 56 +++++++++++++++++++ 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/binary/BinaryClassificationMetricComputers.scala b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/binary/BinaryClassificationMetricComputers.scala index 562663ad36b40..3b573db115c25 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/binary/BinaryClassificationMetricComputers.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/binary/BinaryClassificationMetricComputers.scala @@ -27,19 +27,31 @@ private[evaluation] trait BinaryClassificationMetricComputer extends Serializabl /** Precision. */ private[evaluation] object Precision extends BinaryClassificationMetricComputer { override def apply(c: BinaryConfusionMatrix): Double = - c.numTruePositives.toDouble / (c.numTruePositives + c.numFalsePositives) + if (c.numTruePositives + c.numFalsePositives == 0) { + 0.0 + } else { + c.numTruePositives.toDouble / (c.numTruePositives + c.numFalsePositives) + } } /** False positive rate. */ private[evaluation] object FalsePositiveRate extends BinaryClassificationMetricComputer { override def apply(c: BinaryConfusionMatrix): Double = - c.numFalsePositives.toDouble / c.numNegatives + if (c.numNegatives == 0) { + 0.0 + } else { + c.numFalsePositives.toDouble / c.numNegatives + } } /** Recall. */ private[evaluation] object Recall extends BinaryClassificationMetricComputer { override def apply(c: BinaryConfusionMatrix): Double = - c.numTruePositives.toDouble / c.numPositives + if (c.numPositives == 0) { + 0.0 + } else { + c.numTruePositives.toDouble / c.numPositives + } } /** @@ -52,6 +64,10 @@ private[evaluation] case class FMeasure(beta: Double) extends BinaryClassificati override def apply(c: BinaryConfusionMatrix): Double = { val precision = Precision(c) val recall = Recall(c) - (1.0 + beta2) * (precision * recall) / (beta2 * precision + recall) + if (precision + recall == 0) { + 0.0 + } else { + (1.0 + beta2) * (precision * recall) / (beta2 * precision + recall) + } } } diff --git a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala index a733f88b60b80..a7f2e63b77ba9 100644 --- a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala @@ -59,4 +59,60 @@ class BinaryClassificationMetricsSuite extends FunSuite with LocalSparkContext { assert(metrics.precisionByThreshold().collect().zip(threshold.zip(precision)).forall(cond2)) assert(metrics.recallByThreshold().collect().zip(threshold.zip(recall)).forall(cond2)) } + + test("binary evaluation metrics for All Positive RDD") { + val scoreAndLabels = sc.parallelize(Seq((0.5, 1.0)), 2) + val metrics: BinaryClassificationMetrics = new BinaryClassificationMetrics(scoreAndLabels) + + val threshold = Seq(0.5) + val precision = Seq(1.0) + val recall = Seq(1.0) + val fpr = Seq(0.0) + val rocCurve = Seq((0.0, 0.0)) ++ fpr.zip(recall) ++ Seq((1.0, 1.0)) + val pr = recall.zip(precision) + val prCurve = Seq((0.0, 1.0)) ++ pr + val f1 = pr.map { case (r, p) => 2.0 * (p * r) / (p + r)} + val f2 = pr.map { case (r, p) => 5.0 * (p * r) / (4.0 * p + r)} + + assert(metrics.thresholds().collect().zip(threshold).forall(cond1)) + assert(metrics.roc().collect().zip(rocCurve).forall(cond2)) + assert(metrics.areaUnderROC() ~== AreaUnderCurve.of(rocCurve) absTol 1E-5) + assert(metrics.pr().collect().zip(prCurve).forall(cond2)) + assert(metrics.areaUnderPR() ~== AreaUnderCurve.of(prCurve) absTol 1E-5) + assert(metrics.fMeasureByThreshold().collect().zip(threshold.zip(f1)).forall(cond2)) + assert(metrics.fMeasureByThreshold(2.0).collect().zip(threshold.zip(f2)).forall(cond2)) + assert(metrics.precisionByThreshold().collect().zip(threshold.zip(precision)).forall(cond2)) + assert(metrics.recallByThreshold().collect().zip(threshold.zip(recall)).forall(cond2)) + } + + test("binary evaluation metrics for All Negative RDD") { + val scoreAndLabels = sc.parallelize(Seq((0.5, 0.0)), 2) + val metrics: BinaryClassificationMetrics = new BinaryClassificationMetrics(scoreAndLabels) + + val threshold = Seq(0.5) + val precision = Seq(0.0) + val recall = Seq(0.0) + val fpr = Seq(1.0) + val rocCurve = Seq((0.0, 0.0)) ++ fpr.zip(recall) ++ Seq((1.0, 1.0)) + val pr = recall.zip(precision) + val prCurve = Seq((0.0, 1.0)) ++ pr + val f1 = pr.map { + case (0,0) => 0.0 + case (r, p) => 2.0 * (p * r) / (p + r) + } + val f2 = pr.map { + case (0,0) => 0.0 + case (r, p) => 5.0 * (p * r) / (4.0 * p + r) + } + + assert(metrics.thresholds().collect().zip(threshold).forall(cond1)) + assert(metrics.roc().collect().zip(rocCurve).forall(cond2)) + assert(metrics.areaUnderROC() ~== AreaUnderCurve.of(rocCurve) absTol 1E-5) + assert(metrics.pr().collect().zip(prCurve).forall(cond2)) + assert(metrics.areaUnderPR() ~== AreaUnderCurve.of(prCurve) absTol 1E-5) + assert(metrics.fMeasureByThreshold().collect().zip(threshold.zip(f1)).forall(cond2)) + assert(metrics.fMeasureByThreshold(2.0).collect().zip(threshold.zip(f2)).forall(cond2)) + assert(metrics.precisionByThreshold().collect().zip(threshold.zip(precision)).forall(cond2)) + assert(metrics.recallByThreshold().collect().zip(threshold.zip(recall)).forall(cond2)) + } } From f411e7034800d1eead88de2025983250df70a775 Mon Sep 17 00:00:00 2001 From: Andrew Bullen Date: Mon, 10 Nov 2014 12:04:35 -0800 Subject: [PATCH 2/5] [SPARK-4256] Define precision as 1.0 when there are no positive examples; update code formatting per pull request comments --- .../BinaryClassificationMetricComputers.scala | 25 +++++++++++-------- .../BinaryClassificationMetricsSuite.scala | 8 +++--- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/binary/BinaryClassificationMetricComputers.scala b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/binary/BinaryClassificationMetricComputers.scala index 3b573db115c25..be3319d60ce25 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/binary/BinaryClassificationMetricComputers.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/binary/BinaryClassificationMetricComputers.scala @@ -24,38 +24,43 @@ private[evaluation] trait BinaryClassificationMetricComputer extends Serializabl def apply(c: BinaryConfusionMatrix): Double } -/** Precision. */ +/** Precision. Defined as 1.0 when there are no positive examples. */ private[evaluation] object Precision extends BinaryClassificationMetricComputer { - override def apply(c: BinaryConfusionMatrix): Double = - if (c.numTruePositives + c.numFalsePositives == 0) { - 0.0 + override def apply(c: BinaryConfusionMatrix): Double = { + val totalPositives = c.numTruePositives + c.numFalsePositives + if (totalPositives == 0) { + 1.0 } else { - c.numTruePositives.toDouble / (c.numTruePositives + c.numFalsePositives) + c.numTruePositives.toDouble / totalPositives } + } } -/** False positive rate. */ +/** False positive rate. Defined as 0.0 when there are no negative examples. */ private[evaluation] object FalsePositiveRate extends BinaryClassificationMetricComputer { - override def apply(c: BinaryConfusionMatrix): Double = + override def apply(c: BinaryConfusionMatrix): Double = { if (c.numNegatives == 0) { 0.0 } else { c.numFalsePositives.toDouble / c.numNegatives } + } } -/** Recall. */ +/** Recall. Defined as 0.0 when there are no positive examples. */ private[evaluation] object Recall extends BinaryClassificationMetricComputer { - override def apply(c: BinaryConfusionMatrix): Double = + override def apply(c: BinaryConfusionMatrix): Double = { if (c.numPositives == 0) { 0.0 } else { c.numTruePositives.toDouble / c.numPositives } + } } /** - * F-Measure. + * F-Measure. Defined as 0 if both precision and recall are 0. EG in the case that all examples + * are false positives. * @param beta the beta constant in F-Measure * @see http://en.wikipedia.org/wiki/F1_score */ diff --git a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala index a7f2e63b77ba9..54d481940bca6 100644 --- a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala @@ -61,7 +61,7 @@ class BinaryClassificationMetricsSuite extends FunSuite with LocalSparkContext { } test("binary evaluation metrics for All Positive RDD") { - val scoreAndLabels = sc.parallelize(Seq((0.5, 1.0)), 2) + val scoreAndLabels = sc.parallelize(Seq((0.5, 1.0), (0.5, 1.0)), 2) val metrics: BinaryClassificationMetrics = new BinaryClassificationMetrics(scoreAndLabels) val threshold = Seq(0.5) @@ -86,7 +86,7 @@ class BinaryClassificationMetricsSuite extends FunSuite with LocalSparkContext { } test("binary evaluation metrics for All Negative RDD") { - val scoreAndLabels = sc.parallelize(Seq((0.5, 0.0)), 2) + val scoreAndLabels = sc.parallelize(Seq((0.5, 0.0), (0.5, 0.0)), 2) val metrics: BinaryClassificationMetrics = new BinaryClassificationMetrics(scoreAndLabels) val threshold = Seq(0.5) @@ -97,11 +97,11 @@ class BinaryClassificationMetricsSuite extends FunSuite with LocalSparkContext { val pr = recall.zip(precision) val prCurve = Seq((0.0, 1.0)) ++ pr val f1 = pr.map { - case (0,0) => 0.0 + case (0, 0) => 0.0 case (r, p) => 2.0 * (p * r) / (p + r) } val f2 = pr.map { - case (0,0) => 0.0 + case (0, 0) => 0.0 case (r, p) => 5.0 * (p * r) / (4.0 * p + r) } From 4d2f79ae95a99086ac4b84519094b571bc1e2011 Mon Sep 17 00:00:00 2001 From: Andrew Bullen Date: Mon, 10 Nov 2014 13:39:16 -0800 Subject: [PATCH 3/5] [SPARK-4256] Refactor classification metrics tests - extract comparison functions in test --- .../BinaryClassificationMetricsSuite.scala | 54 +++++++++++-------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala index 54d481940bca6..f3bcdb3a93b76 100644 --- a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala @@ -24,11 +24,19 @@ import org.apache.spark.mllib.util.TestingUtils._ class BinaryClassificationMetricsSuite extends FunSuite with LocalSparkContext { - def cond1(x: (Double, Double)): Boolean = x._1 ~= (x._2) absTol 1E-5 + def areWithinEpsilon(x: (Double, Double)): Boolean = x._1 ~= (x._2) absTol 1E-5 - def cond2(x: ((Double, Double), (Double, Double))): Boolean = + def pairsWithinEpsilon(x: ((Double, Double), (Double, Double))): Boolean = (x._1._1 ~= x._2._1 absTol 1E-5) && (x._1._2 ~= x._2._2 absTol 1E-5) + private def assertSequencesMatch(left: Seq[Double], right: Seq[Double]): Unit = { + assert(left.zip(right).forall(areWithinEpsilon)) + } + + private def assertTupleSequencesMatch(left: Seq[(Double, Double)], right: Seq[(Double, Double)]): Unit = { + assert(left.zip(right).forall(pairsWithinEpsilon)) + } + test("binary evaluation metrics") { val scoreAndLabels = sc.parallelize( Seq((0.1, 0.0), (0.1, 1.0), (0.4, 0.0), (0.6, 0.0), (0.6, 1.0), (0.6, 1.0), (0.8, 1.0)), 2) @@ -49,15 +57,15 @@ class BinaryClassificationMetricsSuite extends FunSuite with LocalSparkContext { val f1 = pr.map { case (r, p) => 2.0 * (p * r) / (p + r)} val f2 = pr.map { case (r, p) => 5.0 * (p * r) / (4.0 * p + r)} - assert(metrics.thresholds().collect().zip(threshold).forall(cond1)) - assert(metrics.roc().collect().zip(rocCurve).forall(cond2)) + assertSequencesMatch(metrics.thresholds().collect(), threshold) + assertTupleSequencesMatch(metrics.roc().collect(), rocCurve) assert(metrics.areaUnderROC() ~== AreaUnderCurve.of(rocCurve) absTol 1E-5) - assert(metrics.pr().collect().zip(prCurve).forall(cond2)) + assertTupleSequencesMatch(metrics.pr().collect(), prCurve) assert(metrics.areaUnderPR() ~== AreaUnderCurve.of(prCurve) absTol 1E-5) - assert(metrics.fMeasureByThreshold().collect().zip(threshold.zip(f1)).forall(cond2)) - assert(metrics.fMeasureByThreshold(2.0).collect().zip(threshold.zip(f2)).forall(cond2)) - assert(metrics.precisionByThreshold().collect().zip(threshold.zip(precision)).forall(cond2)) - assert(metrics.recallByThreshold().collect().zip(threshold.zip(recall)).forall(cond2)) + assertTupleSequencesMatch(metrics.fMeasureByThreshold().collect(), threshold.zip(f1)) + assertTupleSequencesMatch(metrics.fMeasureByThreshold(2.0).collect(), threshold.zip(f2)) + assertTupleSequencesMatch(metrics.precisionByThreshold().collect(), threshold.zip(precision)) + assertTupleSequencesMatch(metrics.recallByThreshold().collect(), threshold.zip(recall)) } test("binary evaluation metrics for All Positive RDD") { @@ -74,15 +82,15 @@ class BinaryClassificationMetricsSuite extends FunSuite with LocalSparkContext { val f1 = pr.map { case (r, p) => 2.0 * (p * r) / (p + r)} val f2 = pr.map { case (r, p) => 5.0 * (p * r) / (4.0 * p + r)} - assert(metrics.thresholds().collect().zip(threshold).forall(cond1)) - assert(metrics.roc().collect().zip(rocCurve).forall(cond2)) + assertSequencesMatch(metrics.thresholds().collect(), threshold) + assertTupleSequencesMatch(metrics.roc().collect(), rocCurve) assert(metrics.areaUnderROC() ~== AreaUnderCurve.of(rocCurve) absTol 1E-5) - assert(metrics.pr().collect().zip(prCurve).forall(cond2)) + assertTupleSequencesMatch(metrics.pr().collect(), prCurve) assert(metrics.areaUnderPR() ~== AreaUnderCurve.of(prCurve) absTol 1E-5) - assert(metrics.fMeasureByThreshold().collect().zip(threshold.zip(f1)).forall(cond2)) - assert(metrics.fMeasureByThreshold(2.0).collect().zip(threshold.zip(f2)).forall(cond2)) - assert(metrics.precisionByThreshold().collect().zip(threshold.zip(precision)).forall(cond2)) - assert(metrics.recallByThreshold().collect().zip(threshold.zip(recall)).forall(cond2)) + assertTupleSequencesMatch(metrics.fMeasureByThreshold().collect(), threshold.zip(f1)) + assertTupleSequencesMatch(metrics.fMeasureByThreshold(2.0).collect(), threshold.zip(f2)) + assertTupleSequencesMatch(metrics.precisionByThreshold().collect(), threshold.zip(precision)) + assertTupleSequencesMatch(metrics.recallByThreshold().collect(), threshold.zip(recall)) } test("binary evaluation metrics for All Negative RDD") { @@ -105,14 +113,14 @@ class BinaryClassificationMetricsSuite extends FunSuite with LocalSparkContext { case (r, p) => 5.0 * (p * r) / (4.0 * p + r) } - assert(metrics.thresholds().collect().zip(threshold).forall(cond1)) - assert(metrics.roc().collect().zip(rocCurve).forall(cond2)) + assertSequencesMatch(metrics.thresholds().collect(), threshold) + assertTupleSequencesMatch(metrics.roc().collect(), rocCurve) assert(metrics.areaUnderROC() ~== AreaUnderCurve.of(rocCurve) absTol 1E-5) - assert(metrics.pr().collect().zip(prCurve).forall(cond2)) + assertTupleSequencesMatch(metrics.pr().collect(), prCurve) assert(metrics.areaUnderPR() ~== AreaUnderCurve.of(prCurve) absTol 1E-5) - assert(metrics.fMeasureByThreshold().collect().zip(threshold.zip(f1)).forall(cond2)) - assert(metrics.fMeasureByThreshold(2.0).collect().zip(threshold.zip(f2)).forall(cond2)) - assert(metrics.precisionByThreshold().collect().zip(threshold.zip(precision)).forall(cond2)) - assert(metrics.recallByThreshold().collect().zip(threshold.zip(recall)).forall(cond2)) + assertTupleSequencesMatch(metrics.fMeasureByThreshold().collect(), threshold.zip(f1)) + assertTupleSequencesMatch(metrics.fMeasureByThreshold(2.0).collect(), threshold.zip(f2)) + assertTupleSequencesMatch(metrics.precisionByThreshold().collect(), threshold.zip(precision)) + assertTupleSequencesMatch(metrics.recallByThreshold().collect(), threshold.zip(recall)) } } From 36b053381f3c70da8a88208ddcb9617f1f5139cf Mon Sep 17 00:00:00 2001 From: Andrew Bullen Date: Mon, 10 Nov 2014 13:55:56 -0800 Subject: [PATCH 4/5] [SYMAN-4256] Extract BinaryClassificationMetricsSuite assertions into private method --- .../BinaryClassificationMetricsSuite.scala | 84 +++++++++---------- 1 file changed, 40 insertions(+), 44 deletions(-) diff --git a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala index f3bcdb3a93b76..0b033484c1a9d 100644 --- a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala @@ -24,9 +24,9 @@ import org.apache.spark.mllib.util.TestingUtils._ class BinaryClassificationMetricsSuite extends FunSuite with LocalSparkContext { - def areWithinEpsilon(x: (Double, Double)): Boolean = x._1 ~= (x._2) absTol 1E-5 + private def areWithinEpsilon(x: (Double, Double)): Boolean = x._1 ~= (x._2) absTol 1E-5 - def pairsWithinEpsilon(x: ((Double, Double), (Double, Double))): Boolean = + private def pairsWithinEpsilon(x: ((Double, Double), (Double, Double))): Boolean = (x._1._1 ~= x._2._1 absTol 1E-5) && (x._1._2 ~= x._2._2 absTol 1E-5) private def assertSequencesMatch(left: Seq[Double], right: Seq[Double]): Unit = { @@ -37,72 +37,76 @@ class BinaryClassificationMetricsSuite extends FunSuite with LocalSparkContext { assert(left.zip(right).forall(pairsWithinEpsilon)) } + private def validateMetrics(metrics: BinaryClassificationMetrics, + expectedThresholds: Seq[Double], + expectedROCCurve: Seq[(Double, Double)], + expectedPRCurve: Seq[(Double, Double)], + expectedFMeasures1: Seq[Double], + expectedFmeasures2: Seq[Double], + expectedPrecisions: Seq[Double], + expectedRecalls: Seq[Double]) = { + + assertSequencesMatch(metrics.thresholds().collect(), expectedThresholds) + assertTupleSequencesMatch(metrics.roc().collect(), expectedROCCurve) + assert(metrics.areaUnderROC() ~== AreaUnderCurve.of(expectedROCCurve) absTol 1E-5) + assertTupleSequencesMatch(metrics.pr().collect(), expectedPRCurve) + assert(metrics.areaUnderPR() ~== AreaUnderCurve.of(expectedPRCurve) absTol 1E-5) + assertTupleSequencesMatch(metrics.fMeasureByThreshold().collect(), expectedThresholds.zip(expectedFMeasures1)) + assertTupleSequencesMatch(metrics.fMeasureByThreshold(2.0).collect(), expectedThresholds.zip(expectedFmeasures2)) + assertTupleSequencesMatch(metrics.precisionByThreshold().collect(), expectedThresholds.zip(expectedPrecisions)) + assertTupleSequencesMatch(metrics.recallByThreshold().collect(), expectedThresholds.zip(expectedRecalls)) + } + test("binary evaluation metrics") { val scoreAndLabels = sc.parallelize( Seq((0.1, 0.0), (0.1, 1.0), (0.4, 0.0), (0.6, 0.0), (0.6, 1.0), (0.6, 1.0), (0.8, 1.0)), 2) val metrics = new BinaryClassificationMetrics(scoreAndLabels) - val threshold = Seq(0.8, 0.6, 0.4, 0.1) + val thresholds = Seq(0.8, 0.6, 0.4, 0.1) val numTruePositives = Seq(1, 3, 3, 4) val numFalsePositives = Seq(0, 1, 2, 3) val numPositives = 4 val numNegatives = 3 - val precision = numTruePositives.zip(numFalsePositives).map { case (t, f) => + val precisions = numTruePositives.zip(numFalsePositives).map { case (t, f) => t.toDouble / (t + f) } - val recall = numTruePositives.map(t => t.toDouble / numPositives) + val recalls = numTruePositives.map(t => t.toDouble / numPositives) val fpr = numFalsePositives.map(f => f.toDouble / numNegatives) - val rocCurve = Seq((0.0, 0.0)) ++ fpr.zip(recall) ++ Seq((1.0, 1.0)) - val pr = recall.zip(precision) + val rocCurve = Seq((0.0, 0.0)) ++ fpr.zip(recalls) ++ Seq((1.0, 1.0)) + val pr = recalls.zip(precisions) val prCurve = Seq((0.0, 1.0)) ++ pr val f1 = pr.map { case (r, p) => 2.0 * (p * r) / (p + r)} val f2 = pr.map { case (r, p) => 5.0 * (p * r) / (4.0 * p + r)} - assertSequencesMatch(metrics.thresholds().collect(), threshold) - assertTupleSequencesMatch(metrics.roc().collect(), rocCurve) - assert(metrics.areaUnderROC() ~== AreaUnderCurve.of(rocCurve) absTol 1E-5) - assertTupleSequencesMatch(metrics.pr().collect(), prCurve) - assert(metrics.areaUnderPR() ~== AreaUnderCurve.of(prCurve) absTol 1E-5) - assertTupleSequencesMatch(metrics.fMeasureByThreshold().collect(), threshold.zip(f1)) - assertTupleSequencesMatch(metrics.fMeasureByThreshold(2.0).collect(), threshold.zip(f2)) - assertTupleSequencesMatch(metrics.precisionByThreshold().collect(), threshold.zip(precision)) - assertTupleSequencesMatch(metrics.recallByThreshold().collect(), threshold.zip(recall)) + validateMetrics(metrics, thresholds, rocCurve, prCurve, f1, f2, precisions, recalls) } test("binary evaluation metrics for All Positive RDD") { val scoreAndLabels = sc.parallelize(Seq((0.5, 1.0), (0.5, 1.0)), 2) val metrics: BinaryClassificationMetrics = new BinaryClassificationMetrics(scoreAndLabels) - val threshold = Seq(0.5) - val precision = Seq(1.0) - val recall = Seq(1.0) + val thresholds = Seq(0.5) + val precisions = Seq(1.0) + val recalls = Seq(1.0) val fpr = Seq(0.0) - val rocCurve = Seq((0.0, 0.0)) ++ fpr.zip(recall) ++ Seq((1.0, 1.0)) - val pr = recall.zip(precision) + val rocCurve = Seq((0.0, 0.0)) ++ fpr.zip(recalls) ++ Seq((1.0, 1.0)) + val pr = recalls.zip(precisions) val prCurve = Seq((0.0, 1.0)) ++ pr val f1 = pr.map { case (r, p) => 2.0 * (p * r) / (p + r)} val f2 = pr.map { case (r, p) => 5.0 * (p * r) / (4.0 * p + r)} - assertSequencesMatch(metrics.thresholds().collect(), threshold) - assertTupleSequencesMatch(metrics.roc().collect(), rocCurve) - assert(metrics.areaUnderROC() ~== AreaUnderCurve.of(rocCurve) absTol 1E-5) - assertTupleSequencesMatch(metrics.pr().collect(), prCurve) - assert(metrics.areaUnderPR() ~== AreaUnderCurve.of(prCurve) absTol 1E-5) - assertTupleSequencesMatch(metrics.fMeasureByThreshold().collect(), threshold.zip(f1)) - assertTupleSequencesMatch(metrics.fMeasureByThreshold(2.0).collect(), threshold.zip(f2)) - assertTupleSequencesMatch(metrics.precisionByThreshold().collect(), threshold.zip(precision)) - assertTupleSequencesMatch(metrics.recallByThreshold().collect(), threshold.zip(recall)) + validateMetrics(metrics, thresholds, rocCurve, prCurve, f1, f2, precisions, recalls) } test("binary evaluation metrics for All Negative RDD") { val scoreAndLabels = sc.parallelize(Seq((0.5, 0.0), (0.5, 0.0)), 2) val metrics: BinaryClassificationMetrics = new BinaryClassificationMetrics(scoreAndLabels) - val threshold = Seq(0.5) - val precision = Seq(0.0) - val recall = Seq(0.0) + val thresholds = Seq(0.5) + val precisions = Seq(0.0) + val recalls = Seq(0.0) val fpr = Seq(1.0) - val rocCurve = Seq((0.0, 0.0)) ++ fpr.zip(recall) ++ Seq((1.0, 1.0)) - val pr = recall.zip(precision) + val rocCurve = Seq((0.0, 0.0)) ++ fpr.zip(recalls) ++ Seq((1.0, 1.0)) + val pr = recalls.zip(precisions) val prCurve = Seq((0.0, 1.0)) ++ pr val f1 = pr.map { case (0, 0) => 0.0 @@ -113,14 +117,6 @@ class BinaryClassificationMetricsSuite extends FunSuite with LocalSparkContext { case (r, p) => 5.0 * (p * r) / (4.0 * p + r) } - assertSequencesMatch(metrics.thresholds().collect(), threshold) - assertTupleSequencesMatch(metrics.roc().collect(), rocCurve) - assert(metrics.areaUnderROC() ~== AreaUnderCurve.of(rocCurve) absTol 1E-5) - assertTupleSequencesMatch(metrics.pr().collect(), prCurve) - assert(metrics.areaUnderPR() ~== AreaUnderCurve.of(prCurve) absTol 1E-5) - assertTupleSequencesMatch(metrics.fMeasureByThreshold().collect(), threshold.zip(f1)) - assertTupleSequencesMatch(metrics.fMeasureByThreshold(2.0).collect(), threshold.zip(f2)) - assertTupleSequencesMatch(metrics.precisionByThreshold().collect(), threshold.zip(precision)) - assertTupleSequencesMatch(metrics.recallByThreshold().collect(), threshold.zip(recall)) + validateMetrics(metrics, thresholds, rocCurve, prCurve, f1, f2, precisions, recalls) } } From c2bf2b19315d75f7573b97e043fb2089945ace50 Mon Sep 17 00:00:00 2001 From: Andrew Bullen Date: Wed, 12 Nov 2014 14:02:56 -0800 Subject: [PATCH 5/5] [SPARK-4256] Update Code formatting for BinaryClassificationMetricsSpec --- .../BinaryClassificationMetricsSuite.scala | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala index 0b033484c1a9d..ac77f0bb5d8f7 100644 --- a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetricsSuite.scala @@ -30,31 +30,36 @@ class BinaryClassificationMetricsSuite extends FunSuite with LocalSparkContext { (x._1._1 ~= x._2._1 absTol 1E-5) && (x._1._2 ~= x._2._2 absTol 1E-5) private def assertSequencesMatch(left: Seq[Double], right: Seq[Double]): Unit = { - assert(left.zip(right).forall(areWithinEpsilon)) + assert(left.zip(right).forall(areWithinEpsilon)) } - private def assertTupleSequencesMatch(left: Seq[(Double, Double)], right: Seq[(Double, Double)]): Unit = { + private def assertTupleSequencesMatch(left: Seq[(Double, Double)], + right: Seq[(Double, Double)]): Unit = { assert(left.zip(right).forall(pairsWithinEpsilon)) } private def validateMetrics(metrics: BinaryClassificationMetrics, - expectedThresholds: Seq[Double], - expectedROCCurve: Seq[(Double, Double)], - expectedPRCurve: Seq[(Double, Double)], - expectedFMeasures1: Seq[Double], - expectedFmeasures2: Seq[Double], - expectedPrecisions: Seq[Double], - expectedRecalls: Seq[Double]) = { + expectedThresholds: Seq[Double], + expectedROCCurve: Seq[(Double, Double)], + expectedPRCurve: Seq[(Double, Double)], + expectedFMeasures1: Seq[Double], + expectedFmeasures2: Seq[Double], + expectedPrecisions: Seq[Double], + expectedRecalls: Seq[Double]) = { assertSequencesMatch(metrics.thresholds().collect(), expectedThresholds) assertTupleSequencesMatch(metrics.roc().collect(), expectedROCCurve) assert(metrics.areaUnderROC() ~== AreaUnderCurve.of(expectedROCCurve) absTol 1E-5) assertTupleSequencesMatch(metrics.pr().collect(), expectedPRCurve) assert(metrics.areaUnderPR() ~== AreaUnderCurve.of(expectedPRCurve) absTol 1E-5) - assertTupleSequencesMatch(metrics.fMeasureByThreshold().collect(), expectedThresholds.zip(expectedFMeasures1)) - assertTupleSequencesMatch(metrics.fMeasureByThreshold(2.0).collect(), expectedThresholds.zip(expectedFmeasures2)) - assertTupleSequencesMatch(metrics.precisionByThreshold().collect(), expectedThresholds.zip(expectedPrecisions)) - assertTupleSequencesMatch(metrics.recallByThreshold().collect(), expectedThresholds.zip(expectedRecalls)) + assertTupleSequencesMatch(metrics.fMeasureByThreshold().collect(), + expectedThresholds.zip(expectedFMeasures1)) + assertTupleSequencesMatch(metrics.fMeasureByThreshold(2.0).collect(), + expectedThresholds.zip(expectedFmeasures2)) + assertTupleSequencesMatch(metrics.precisionByThreshold().collect(), + expectedThresholds.zip(expectedPrecisions)) + assertTupleSequencesMatch(metrics.recallByThreshold().collect(), + expectedThresholds.zip(expectedRecalls)) } test("binary evaluation metrics") { @@ -80,9 +85,9 @@ class BinaryClassificationMetricsSuite extends FunSuite with LocalSparkContext { validateMetrics(metrics, thresholds, rocCurve, prCurve, f1, f2, precisions, recalls) } - test("binary evaluation metrics for All Positive RDD") { + test("binary evaluation metrics for RDD where all examples have positive label") { val scoreAndLabels = sc.parallelize(Seq((0.5, 1.0), (0.5, 1.0)), 2) - val metrics: BinaryClassificationMetrics = new BinaryClassificationMetrics(scoreAndLabels) + val metrics = new BinaryClassificationMetrics(scoreAndLabels) val thresholds = Seq(0.5) val precisions = Seq(1.0) @@ -97,9 +102,9 @@ class BinaryClassificationMetricsSuite extends FunSuite with LocalSparkContext { validateMetrics(metrics, thresholds, rocCurve, prCurve, f1, f2, precisions, recalls) } - test("binary evaluation metrics for All Negative RDD") { + test("binary evaluation metrics for RDD where all examples have negative label") { val scoreAndLabels = sc.parallelize(Seq((0.5, 0.0), (0.5, 0.0)), 2) - val metrics: BinaryClassificationMetrics = new BinaryClassificationMetrics(scoreAndLabels) + val metrics = new BinaryClassificationMetrics(scoreAndLabels) val thresholds = Seq(0.5) val precisions = Seq(0.0)