Skip to content

Move evaluation results to separate classes #1663

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Nov 29, 2018
Merged

Move evaluation results to separate classes #1663

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
9f5a8b7
Move evaluation results to stand alone metrics classes
robosek Nov 17, 2018
af8b8e1
Update metrics types
robosek Nov 17, 2018
c8eb287
Uppercase 'C' in MultiClassClassifierEvaluator and Metrics to keep co…
robosek Nov 17, 2018
ab3ddee
Fix metrics types
robosek Nov 17, 2018
0fdc184
Merge branch 'master' into metrics-classes
robosek Nov 19, 2018
3a94abd
Change namespace to Microsoft.ML.Data
robosek Nov 19, 2018
61b2941
Adjust metrics namespaces
robosek Nov 19, 2018
bd04950
XMl encode apostrophes
robosek Nov 20, 2018
db3d08f
Merge branch 'master' into metrics-classes
robosek Nov 20, 2018
9561045
Fix conflicts and merge master
robosek Nov 28, 2018
ece2b6e
Use metric types
robosek Nov 28, 2018
ce5b416
merge submodules
robosek Nov 28, 2018
33d35f4
Fix descriptions
robosek Nov 28, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
161 changes: 6 additions & 155 deletions src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -790,155 +790,6 @@ private void ComputePrCurves()
}
}

/// <summary>
/// Evaluation results for binary classifiers, excluding probabilistic metrics.
/// </summary>
public class Result
{
/// <summary>
/// Gets the area under the ROC curve.
/// </summary>
/// <remarks>
/// The area under the ROC curve is equal to the probability that the classifier ranks
/// a randomly chosen positive instance higher than a randomly chosen negative one
/// (assuming 'positive' ranks higher than 'negative').
/// </remarks>
public double Auc { get; }

/// <summary>
/// Gets the accuracy of a classifier which is the proportion of correct predictions in the test set.
/// </summary>
public double Accuracy { get; }

/// <summary>
/// Gets the positive precision of a classifier which is the proportion of correctly predicted
/// positive instances among all the positive predictions (i.e., the number of positive instances
/// predicted as positive, divided by the total number of instances predicted as positive).
/// </summary>
public double PositivePrecision { get; }

/// <summary>
/// Gets the positive recall of a classifier which is the proportion of correctly predicted
/// positive instances among all the positive instances (i.e., the number of positive instances
/// predicted as positive, divided by the total number of positive instances).
/// </summary>
public double PositiveRecall { get; private set; }

/// <summary>
/// Gets the negative precision of a classifier which is the proportion of correctly predicted
/// negative instances among all the negative predictions (i.e., the number of negative instances
/// predicted as negative, divided by the total number of instances predicted as negative).
/// </summary>
public double NegativePrecision { get; }

/// <summary>
/// Gets the negative recall of a classifier which is the proportion of correctly predicted
/// negative instances among all the negative instances (i.e., the number of negative instances
/// predicted as negative, divided by the total number of negative instances).
/// </summary>
public double NegativeRecall { get; }

/// <summary>
/// Gets the F1 score of the classifier.
/// </summary>
/// <remarks>
/// F1 score is the harmonic mean of precision and recall: 2 * precision * recall / (precision + recall).
/// </remarks>
public double F1Score { get; }

/// <summary>
/// Gets the area under the precision/recall curve of the classifier.
/// </summary>
/// <remarks>
/// The area under the precision/recall curve is a single number summary of the information in the
/// precision/recall curve. It is increasingly used in the machine learning community, particularly
/// for imbalanced datasets where one class is observed more frequently than the other. On these
/// datasets, AUPRC can highlight performance differences that are lost with AUC.
/// </remarks>
public double Auprc { get; }

protected private static T Fetch<T>(IExceptionContext ectx, IRow row, string name)
{
if (!row.Schema.TryGetColumnIndex(name, out int col))
throw ectx.Except($"Could not find column '{name}'");
T val = default;
row.GetGetter<T>(col)(ref val);
return val;
}

internal Result(IExceptionContext ectx, IRow overallResult)
{
double Fetch(string name) => Fetch<double>(ectx, overallResult, name);
Auc = Fetch(BinaryClassifierEvaluator.Auc);
Accuracy = Fetch(BinaryClassifierEvaluator.Accuracy);
PositivePrecision = Fetch(BinaryClassifierEvaluator.PosPrecName);
PositiveRecall = Fetch(BinaryClassifierEvaluator.PosRecallName);
NegativePrecision = Fetch(BinaryClassifierEvaluator.NegPrecName);
NegativeRecall = Fetch(BinaryClassifierEvaluator.NegRecallName);
F1Score = Fetch(BinaryClassifierEvaluator.F1);
Auprc = Fetch(BinaryClassifierEvaluator.AuPrc);
}

[BestFriend]
internal Result(double auc, double accuracy, double positivePrecision, double positiveRecall,
double negativePrecision, double negativeRecall, double f1Score, double auprc)
{
Auc = auc;
Accuracy = accuracy;
PositivePrecision = positivePrecision;
PositiveRecall = positiveRecall;
NegativePrecision = negativePrecision;
NegativeRecall = negativeRecall;
F1Score = f1Score;
Auprc = auprc;
}
}

/// <summary>
/// Evaluation results for binary classifiers, including probabilistic metrics.
/// </summary>
public sealed class CalibratedResult : Result
{
/// <summary>
/// Gets the log-loss of the classifier.
/// </summary>
/// <remarks>
/// The log-loss metric, is computed as follows:
/// LL = - (1/m) * sum( log(p[i]))
/// where m is the number of instances in the test set.
/// p[i] is the probability returned by the classifier if the instance belongs to class 1,
/// and 1 minus the probability returned by the classifier if the instance belongs to class 0.
/// </remarks>
public double LogLoss { get; }

/// <summary>
/// Gets the log-loss reduction (also known as relative log-loss, or reduction in information gain - RIG)
/// of the classifier.
/// </summary>
/// <remarks>
/// The log-loss reduction is scaled relative to a classifier that predicts the prior for every example:
/// (LL(prior) - LL(classifier)) / LL(prior)
/// This metric can be interpreted as the advantage of the classifier over a random prediction.
/// For example, if the RIG equals 20, it can be interpreted as &quot;the probability of a correct prediction is
/// 20% better than random guessing.&quot;
/// </remarks>
public double LogLossReduction { get; }

/// <summary>
/// Gets the test-set entropy (prior Log-Loss/instance) of the classifier.
/// </summary>
public double Entropy { get; }

internal CalibratedResult(IExceptionContext ectx, IRow overallResult)
: base(ectx, overallResult)
{
double Fetch(string name) => Fetch<double>(ectx, overallResult, name);
LogLoss = Fetch(BinaryClassifierEvaluator.LogLoss);
LogLossReduction = Fetch(BinaryClassifierEvaluator.LogLossReduction);
Entropy = Fetch(BinaryClassifierEvaluator.Entropy);
}
}

/// <summary>
/// Evaluates scored binary classification data.
/// </summary>
Expand All @@ -948,7 +799,7 @@ internal CalibratedResult(IExceptionContext ectx, IRow overallResult)
/// <param name="probability">The name of the probability column in <paramref name="data"/>, the calibrated version of <paramref name="score"/>.</param>
/// <param name="predictedLabel">The name of the predicted label column in <paramref name="data"/>.</param>
/// <returns>The evaluation results for these calibrated outputs.</returns>
public CalibratedResult Evaluate(IDataView data, string label, string score, string probability, string predictedLabel)
public CalibratedBinaryClassificationMetrics Evaluate(IDataView data, string label, string score, string probability, string predictedLabel)
{
Host.CheckValue(data, nameof(data));
Host.CheckNonEmpty(label, nameof(label));
Expand All @@ -966,12 +817,12 @@ public CalibratedResult Evaluate(IDataView data, string label, string score, str
Host.Assert(resultDict.ContainsKey(MetricKinds.OverallMetrics));
var overall = resultDict[MetricKinds.OverallMetrics];

CalibratedResult result;
CalibratedBinaryClassificationMetrics result;
using (var cursor = overall.GetRowCursor(i => true))
{
var moved = cursor.MoveNext();
Host.Assert(moved);
result = new CalibratedResult(Host, cursor);
result = new CalibratedBinaryClassificationMetrics(Host, cursor);
moved = cursor.MoveNext();
Host.Assert(!moved);
}
Expand All @@ -987,7 +838,7 @@ public CalibratedResult Evaluate(IDataView data, string label, string score, str
/// <param name="predictedLabel">The name of the predicted label column in <paramref name="data"/>.</param>
/// <returns>The evaluation results for these uncalibrated outputs.</returns>
/// <seealso cref="Evaluate(IDataView, string, string, string)"/>
public Result Evaluate(IDataView data, string label, string score, string predictedLabel)
public BinaryClassificationMetrics Evaluate(IDataView data, string label, string score, string predictedLabel)
{
Host.CheckValue(data, nameof(data));
Host.CheckNonEmpty(label, nameof(label));
Expand All @@ -1003,12 +854,12 @@ public Result Evaluate(IDataView data, string label, string score, string predic
Host.Assert(resultDict.ContainsKey(MetricKinds.OverallMetrics));
var overall = resultDict[MetricKinds.OverallMetrics];

Result result;
BinaryClassificationMetrics result;
using (var cursor = overall.GetRowCursor(i => true))
{
var moved = cursor.MoveNext();
Host.Assert(moved);
result = new Result(Host, cursor);
result = new BinaryClassificationMetrics(Host, cursor);
moved = cursor.MoveNext();
Host.Assert(!moved);
}
Expand Down
46 changes: 3 additions & 43 deletions src/Microsoft.ML.Data/Evaluators/ClusteringEvaluator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ public ClusteringEvaluator(IHostEnvironment env, Arguments args)
/// <param name="label">The name of the optional label column in <paramref name="data"/>.</param>
/// <param name="features">The name of the optional feature column in <paramref name="data"/>.</param>
/// <returns>The evaluation results.</returns>
public Result Evaluate(IDataView data, string score, string label = null, string features = null)
public ClusteringMetrics Evaluate(IDataView data, string score, string label = null, string features = null)
{
Host.CheckValue(data, nameof(data));
Host.CheckNonEmpty(score, nameof(score));
Expand All @@ -82,12 +82,12 @@ public Result Evaluate(IDataView data, string score, string label = null, string
Host.Assert(resultDict.ContainsKey(MetricKinds.OverallMetrics));
var overall = resultDict[MetricKinds.OverallMetrics];

Result result;
ClusteringMetrics result;
using (var cursor = overall.GetRowCursor(i => true))
{
var moved = cursor.MoveNext();
Host.Assert(moved);
result = new Result(Host, cursor, _calculateDbi);
result = new ClusteringMetrics(Host, cursor, _calculateDbi);
moved = cursor.MoveNext();
Host.Assert(!moved);
}
Expand Down Expand Up @@ -559,46 +559,6 @@ private void AssertValid(bool assertGetters)
}
}
}

/// <summary>
/// The metrics generated after evaluating the clustering predictions.
/// </summary>
public sealed class Result
{
/// <summary>
/// Normalized Mutual Information
/// NMI is a measure of the mutual dependence of the variables.
/// <a href="http://en.wikipedia.org/wiki/Mutual_information#Normalized_variants">Normalized variants</a> work on data that already has cluster labels.
/// Its value ranged from 0 to 1, where higher numbers are better.
/// </summary>
public double Nmi { get; }

/// <summary>
/// Average Score. For the K-Means algorithm, the 'score' is the distance from the centroid to the example.
/// The average score is, therefore, a measure of proximity of the examples to cluster centroids.
/// In other words, it's the 'cluster tightness' measure.
/// Note however, that this metric will only decrease if the number of clusters is increased,
/// and in the extreme case (where each distinct example is its own cluster) it will be equal to zero.
/// </summary>
public double AvgMinScore { get; }

/// <summary>
/// <a href="https://en.wikipedia.org/wiki/Davies�Bouldin_index">Davies-Bouldin Index</a>
/// DBI is a measure of the how much scatter is in the cluster and the cluster separation.
/// </summary>
public double Dbi { get; }

internal Result(IExceptionContext ectx, IRow overallResult, bool calculateDbi)
{
double Fetch(string name) => RowCursorUtils.Fetch<double>(ectx, overallResult, name);

Nmi = Fetch(ClusteringEvaluator.Nmi);
AvgMinScore = Fetch(ClusteringEvaluator.AvgMinScore);

if (calculateDbi)
Dbi = Fetch(ClusteringEvaluator.Dbi);
}
}
}

public sealed class ClusteringPerInstanceEvaluator : PerInstanceEvaluatorBase
Expand Down
15 changes: 8 additions & 7 deletions src/Microsoft.ML.Data/Evaluators/EvaluatorStaticExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
// See the LICENSE file in the project root for more information.

using System;
using Microsoft.ML.Data;
using Microsoft.ML.Runtime.Training;
using Microsoft.ML.StaticPipe;
using Microsoft.ML.StaticPipe.Runtime;
Expand All @@ -24,7 +25,7 @@ public static class EvaluatorStaticExtensions
/// <param name="pred">The index delegate for columns from calibrated prediction of a binary classifier.
/// Under typical scenarios, this will just be the same tuple of results returned from the trainer.</param>
/// <returns>The evaluation results for these calibrated outputs.</returns>
public static BinaryClassifierEvaluator.CalibratedResult Evaluate<T>(
public static CalibratedBinaryClassificationMetrics Evaluate<T>(
this BinaryClassificationContext ctx,
DataView<T> data,
Func<T, Scalar<bool>> label,
Expand Down Expand Up @@ -60,7 +61,7 @@ public static BinaryClassifierEvaluator.CalibratedResult Evaluate<T>(
/// <param name="pred">The index delegate for columns from uncalibrated prediction of a binary classifier.
/// Under typical scenarios, this will just be the same tuple of results returned from the trainer.</param>
/// <returns>The evaluation results for these uncalibrated outputs.</returns>
public static BinaryClassifierEvaluator.Result Evaluate<T>(
public static BinaryClassificationMetrics Evaluate<T>(
this BinaryClassificationContext ctx,
DataView<T> data,
Func<T, Scalar<bool>> label,
Expand Down Expand Up @@ -94,7 +95,7 @@ public static BinaryClassifierEvaluator.Result Evaluate<T>(
/// <param name="label">The optional index delegate for the label column.</param>
/// <param name="features">The optional index delegate for the features column.</param>
/// <returns>The evaluation metrics.</returns>
public static ClusteringEvaluator.Result Evaluate<T>(
public static ClusteringMetrics Evaluate<T>(
this ClusteringContext ctx,
DataView<T> data,
Func<T, Vector<float>> score,
Expand Down Expand Up @@ -127,11 +128,11 @@ public static ClusteringEvaluator.Result Evaluate<T>(
/// <param name="label">The index delegate for the label column.</param>
/// <param name="pred">The index delegate for columns from the prediction of a multiclass classifier.
/// Under typical scenarios, this will just be the same tuple of results returned from the trainer.</param>
/// <param name="topK">If given a positive value, the <see cref="MultiClassClassifierEvaluator.Result.TopKAccuracy"/> will be filled with
/// <param name="topK">If given a positive value, the <see cref="MultiClassClassifierMetrics.TopKAccuracy"/> will be filled with
/// the top-K accuracy, that is, the accuracy assuming we consider an example with the correct class within
/// the top-K values as being stored "correctly."</param>
/// <returns>The evaluation metrics.</returns>
public static MultiClassClassifierEvaluator.Result Evaluate<T, TKey>(
public static MultiClassClassifierMetrics Evaluate<T, TKey>(
this MulticlassClassificationContext ctx,
DataView<T> data,
Func<T, Key<uint, TKey>> label,
Expand Down Expand Up @@ -178,7 +179,7 @@ private sealed class TrivialRegressionLossFactory : ISupportRegressionLossFactor
/// <param name="score">The index delegate for predicted score column.</param>
/// <param name="loss">Potentially custom loss function. If left unspecified defaults to <see cref="SquaredLoss"/>.</param>
/// <returns>The evaluation metrics.</returns>
public static RegressionEvaluator.Result Evaluate<T>(
public static RegressionMetrics Evaluate<T>(
this RegressionContext ctx,
DataView<T> data,
Func<T, Scalar<float>> label,
Expand Down Expand Up @@ -212,7 +213,7 @@ public static RegressionEvaluator.Result Evaluate<T>(
/// <param name="groupId">The index delegate for the groupId column. </param>
/// <param name="score">The index delegate for predicted score column.</param>
/// <returns>The evaluation metrics.</returns>
public static RankerEvaluator.Result Evaluate<T, TVal>(
public static RankerMetrics Evaluate<T, TVal>(
this RankingContext ctx,
DataView<T> data,
Func<T, Scalar<float>> label,
Expand Down
Loading