diff --git a/src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs b/src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs index a06e038571..528f24efce 100644 --- a/src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs +++ b/src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs @@ -790,155 +790,6 @@ private void ComputePrCurves() } } - /// - /// Evaluation results for binary classifiers, excluding probabilistic metrics. - /// - public class Result - { - /// - /// Gets the area under the ROC curve. - /// - /// - /// The area under the ROC curve is equal to the probability that the classifier ranks - /// a randomly chosen positive instance higher than a randomly chosen negative one - /// (assuming 'positive' ranks higher than 'negative'). - /// - public double Auc { get; } - - /// - /// Gets the accuracy of a classifier which is the proportion of correct predictions in the test set. - /// - public double Accuracy { get; } - - /// - /// Gets the positive precision of a classifier which is the proportion of correctly predicted - /// positive instances among all the positive predictions (i.e., the number of positive instances - /// predicted as positive, divided by the total number of instances predicted as positive). - /// - public double PositivePrecision { get; } - - /// - /// Gets the positive recall of a classifier which is the proportion of correctly predicted - /// positive instances among all the positive instances (i.e., the number of positive instances - /// predicted as positive, divided by the total number of positive instances). - /// - public double PositiveRecall { get; private set; } - - /// - /// Gets the negative precision of a classifier which is the proportion of correctly predicted - /// negative instances among all the negative predictions (i.e., the number of negative instances - /// predicted as negative, divided by the total number of instances predicted as negative). - /// - public double NegativePrecision { get; } - - /// - /// Gets the negative recall of a classifier which is the proportion of correctly predicted - /// negative instances among all the negative instances (i.e., the number of negative instances - /// predicted as negative, divided by the total number of negative instances). - /// - public double NegativeRecall { get; } - - /// - /// Gets the F1 score of the classifier. - /// - /// - /// F1 score is the harmonic mean of precision and recall: 2 * precision * recall / (precision + recall). - /// - public double F1Score { get; } - - /// - /// Gets the area under the precision/recall curve of the classifier. - /// - /// - /// The area under the precision/recall curve is a single number summary of the information in the - /// precision/recall curve. It is increasingly used in the machine learning community, particularly - /// for imbalanced datasets where one class is observed more frequently than the other. On these - /// datasets, AUPRC can highlight performance differences that are lost with AUC. - /// - public double Auprc { get; } - - protected private static T Fetch(IExceptionContext ectx, IRow row, string name) - { - if (!row.Schema.TryGetColumnIndex(name, out int col)) - throw ectx.Except($"Could not find column '{name}'"); - T val = default; - row.GetGetter(col)(ref val); - return val; - } - - internal Result(IExceptionContext ectx, IRow overallResult) - { - double Fetch(string name) => Fetch(ectx, overallResult, name); - Auc = Fetch(BinaryClassifierEvaluator.Auc); - Accuracy = Fetch(BinaryClassifierEvaluator.Accuracy); - PositivePrecision = Fetch(BinaryClassifierEvaluator.PosPrecName); - PositiveRecall = Fetch(BinaryClassifierEvaluator.PosRecallName); - NegativePrecision = Fetch(BinaryClassifierEvaluator.NegPrecName); - NegativeRecall = Fetch(BinaryClassifierEvaluator.NegRecallName); - F1Score = Fetch(BinaryClassifierEvaluator.F1); - Auprc = Fetch(BinaryClassifierEvaluator.AuPrc); - } - - [BestFriend] - internal Result(double auc, double accuracy, double positivePrecision, double positiveRecall, - double negativePrecision, double negativeRecall, double f1Score, double auprc) - { - Auc = auc; - Accuracy = accuracy; - PositivePrecision = positivePrecision; - PositiveRecall = positiveRecall; - NegativePrecision = negativePrecision; - NegativeRecall = negativeRecall; - F1Score = f1Score; - Auprc = auprc; - } - } - - /// - /// Evaluation results for binary classifiers, including probabilistic metrics. - /// - public sealed class CalibratedResult : Result - { - /// - /// Gets the log-loss of the classifier. - /// - /// - /// The log-loss metric, is computed as follows: - /// LL = - (1/m) * sum( log(p[i])) - /// where m is the number of instances in the test set. - /// p[i] is the probability returned by the classifier if the instance belongs to class 1, - /// and 1 minus the probability returned by the classifier if the instance belongs to class 0. - /// - public double LogLoss { get; } - - /// - /// Gets the log-loss reduction (also known as relative log-loss, or reduction in information gain - RIG) - /// of the classifier. - /// - /// - /// The log-loss reduction is scaled relative to a classifier that predicts the prior for every example: - /// (LL(prior) - LL(classifier)) / LL(prior) - /// This metric can be interpreted as the advantage of the classifier over a random prediction. - /// For example, if the RIG equals 20, it can be interpreted as "the probability of a correct prediction is - /// 20% better than random guessing." - /// - public double LogLossReduction { get; } - - /// - /// Gets the test-set entropy (prior Log-Loss/instance) of the classifier. - /// - public double Entropy { get; } - - internal CalibratedResult(IExceptionContext ectx, IRow overallResult) - : base(ectx, overallResult) - { - double Fetch(string name) => Fetch(ectx, overallResult, name); - LogLoss = Fetch(BinaryClassifierEvaluator.LogLoss); - LogLossReduction = Fetch(BinaryClassifierEvaluator.LogLossReduction); - Entropy = Fetch(BinaryClassifierEvaluator.Entropy); - } - } - /// /// Evaluates scored binary classification data. /// @@ -948,7 +799,7 @@ internal CalibratedResult(IExceptionContext ectx, IRow overallResult) /// The name of the probability column in , the calibrated version of . /// The name of the predicted label column in . /// The evaluation results for these calibrated outputs. - public CalibratedResult Evaluate(IDataView data, string label, string score, string probability, string predictedLabel) + public CalibratedBinaryClassificationMetrics Evaluate(IDataView data, string label, string score, string probability, string predictedLabel) { Host.CheckValue(data, nameof(data)); Host.CheckNonEmpty(label, nameof(label)); @@ -966,12 +817,12 @@ public CalibratedResult Evaluate(IDataView data, string label, string score, str Host.Assert(resultDict.ContainsKey(MetricKinds.OverallMetrics)); var overall = resultDict[MetricKinds.OverallMetrics]; - CalibratedResult result; + CalibratedBinaryClassificationMetrics result; using (var cursor = overall.GetRowCursor(i => true)) { var moved = cursor.MoveNext(); Host.Assert(moved); - result = new CalibratedResult(Host, cursor); + result = new CalibratedBinaryClassificationMetrics(Host, cursor); moved = cursor.MoveNext(); Host.Assert(!moved); } @@ -987,7 +838,7 @@ public CalibratedResult Evaluate(IDataView data, string label, string score, str /// The name of the predicted label column in . /// The evaluation results for these uncalibrated outputs. /// - public Result Evaluate(IDataView data, string label, string score, string predictedLabel) + public BinaryClassificationMetrics Evaluate(IDataView data, string label, string score, string predictedLabel) { Host.CheckValue(data, nameof(data)); Host.CheckNonEmpty(label, nameof(label)); @@ -1003,12 +854,12 @@ public Result Evaluate(IDataView data, string label, string score, string predic Host.Assert(resultDict.ContainsKey(MetricKinds.OverallMetrics)); var overall = resultDict[MetricKinds.OverallMetrics]; - Result result; + BinaryClassificationMetrics result; using (var cursor = overall.GetRowCursor(i => true)) { var moved = cursor.MoveNext(); Host.Assert(moved); - result = new Result(Host, cursor); + result = new BinaryClassificationMetrics(Host, cursor); moved = cursor.MoveNext(); Host.Assert(!moved); } diff --git a/src/Microsoft.ML.Data/Evaluators/ClusteringEvaluator.cs b/src/Microsoft.ML.Data/Evaluators/ClusteringEvaluator.cs index 7e3a39b175..7d015da046 100644 --- a/src/Microsoft.ML.Data/Evaluators/ClusteringEvaluator.cs +++ b/src/Microsoft.ML.Data/Evaluators/ClusteringEvaluator.cs @@ -62,7 +62,7 @@ public ClusteringEvaluator(IHostEnvironment env, Arguments args) /// The name of the optional label column in . /// The name of the optional feature column in . /// The evaluation results. - public Result Evaluate(IDataView data, string score, string label = null, string features = null) + public ClusteringMetrics Evaluate(IDataView data, string score, string label = null, string features = null) { Host.CheckValue(data, nameof(data)); Host.CheckNonEmpty(score, nameof(score)); @@ -82,12 +82,12 @@ public Result Evaluate(IDataView data, string score, string label = null, string Host.Assert(resultDict.ContainsKey(MetricKinds.OverallMetrics)); var overall = resultDict[MetricKinds.OverallMetrics]; - Result result; + ClusteringMetrics result; using (var cursor = overall.GetRowCursor(i => true)) { var moved = cursor.MoveNext(); Host.Assert(moved); - result = new Result(Host, cursor, _calculateDbi); + result = new ClusteringMetrics(Host, cursor, _calculateDbi); moved = cursor.MoveNext(); Host.Assert(!moved); } @@ -559,46 +559,6 @@ private void AssertValid(bool assertGetters) } } } - - /// - /// The metrics generated after evaluating the clustering predictions. - /// - public sealed class Result - { - /// - /// Normalized Mutual Information - /// NMI is a measure of the mutual dependence of the variables. - /// Normalized variants work on data that already has cluster labels. - /// Its value ranged from 0 to 1, where higher numbers are better. - /// - public double Nmi { get; } - - /// - /// Average Score. For the K-Means algorithm, the 'score' is the distance from the centroid to the example. - /// The average score is, therefore, a measure of proximity of the examples to cluster centroids. - /// In other words, it's the 'cluster tightness' measure. - /// Note however, that this metric will only decrease if the number of clusters is increased, - /// and in the extreme case (where each distinct example is its own cluster) it will be equal to zero. - /// - public double AvgMinScore { get; } - - /// - /// Davies-Bouldin Index - /// DBI is a measure of the how much scatter is in the cluster and the cluster separation. - /// - public double Dbi { get; } - - internal Result(IExceptionContext ectx, IRow overallResult, bool calculateDbi) - { - double Fetch(string name) => RowCursorUtils.Fetch(ectx, overallResult, name); - - Nmi = Fetch(ClusteringEvaluator.Nmi); - AvgMinScore = Fetch(ClusteringEvaluator.AvgMinScore); - - if (calculateDbi) - Dbi = Fetch(ClusteringEvaluator.Dbi); - } - } } public sealed class ClusteringPerInstanceEvaluator : PerInstanceEvaluatorBase diff --git a/src/Microsoft.ML.Data/Evaluators/EvaluatorStaticExtensions.cs b/src/Microsoft.ML.Data/Evaluators/EvaluatorStaticExtensions.cs index df58e4d46c..46972dc75d 100644 --- a/src/Microsoft.ML.Data/Evaluators/EvaluatorStaticExtensions.cs +++ b/src/Microsoft.ML.Data/Evaluators/EvaluatorStaticExtensions.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using Microsoft.ML.Data; using Microsoft.ML.Runtime.Training; using Microsoft.ML.StaticPipe; using Microsoft.ML.StaticPipe.Runtime; @@ -24,7 +25,7 @@ public static class EvaluatorStaticExtensions /// The index delegate for columns from calibrated prediction of a binary classifier. /// Under typical scenarios, this will just be the same tuple of results returned from the trainer. /// The evaluation results for these calibrated outputs. - public static BinaryClassifierEvaluator.CalibratedResult Evaluate( + public static CalibratedBinaryClassificationMetrics Evaluate( this BinaryClassificationContext ctx, DataView data, Func> label, @@ -60,7 +61,7 @@ public static BinaryClassifierEvaluator.CalibratedResult Evaluate( /// The index delegate for columns from uncalibrated prediction of a binary classifier. /// Under typical scenarios, this will just be the same tuple of results returned from the trainer. /// The evaluation results for these uncalibrated outputs. - public static BinaryClassifierEvaluator.Result Evaluate( + public static BinaryClassificationMetrics Evaluate( this BinaryClassificationContext ctx, DataView data, Func> label, @@ -94,7 +95,7 @@ public static BinaryClassifierEvaluator.Result Evaluate( /// The optional index delegate for the label column. /// The optional index delegate for the features column. /// The evaluation metrics. - public static ClusteringEvaluator.Result Evaluate( + public static ClusteringMetrics Evaluate( this ClusteringContext ctx, DataView data, Func> score, @@ -127,11 +128,11 @@ public static ClusteringEvaluator.Result Evaluate( /// The index delegate for the label column. /// The index delegate for columns from the prediction of a multiclass classifier. /// Under typical scenarios, this will just be the same tuple of results returned from the trainer. - /// If given a positive value, the will be filled with + /// If given a positive value, the will be filled with /// the top-K accuracy, that is, the accuracy assuming we consider an example with the correct class within /// the top-K values as being stored "correctly." /// The evaluation metrics. - public static MultiClassClassifierEvaluator.Result Evaluate( + public static MultiClassClassifierMetrics Evaluate( this MulticlassClassificationContext ctx, DataView data, Func> label, @@ -178,7 +179,7 @@ private sealed class TrivialRegressionLossFactory : ISupportRegressionLossFactor /// The index delegate for predicted score column. /// Potentially custom loss function. If left unspecified defaults to . /// The evaluation metrics. - public static RegressionEvaluator.Result Evaluate( + public static RegressionMetrics Evaluate( this RegressionContext ctx, DataView data, Func> label, @@ -212,7 +213,7 @@ public static RegressionEvaluator.Result Evaluate( /// The index delegate for the groupId column. /// The index delegate for predicted score column. /// The evaluation metrics. - public static RankerEvaluator.Result Evaluate( + public static RankerMetrics Evaluate( this RankingContext ctx, DataView data, Func> label, diff --git a/src/Microsoft.ML.Data/Evaluators/Metrics/BinaryClassificationMetrics.cs b/src/Microsoft.ML.Data/Evaluators/Metrics/BinaryClassificationMetrics.cs new file mode 100644 index 0000000000..102e43a894 --- /dev/null +++ b/src/Microsoft.ML.Data/Evaluators/Metrics/BinaryClassificationMetrics.cs @@ -0,0 +1,113 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.Data; + +namespace Microsoft.ML.Data +{ + /// + /// Evaluation results for binary classifiers, excluding probabilistic metrics. + /// + public class BinaryClassificationMetrics + { + /// + /// Gets the area under the ROC curve. + /// + /// + /// The area under the ROC curve is equal to the probability that the classifier ranks + /// a randomly chosen positive instance higher than a randomly chosen negative one + /// (assuming 'positive' ranks higher than 'negative'). + /// + public double Auc { get; } + + /// + /// Gets the accuracy of a classifier which is the proportion of correct predictions in the test set. + /// + public double Accuracy { get; } + + /// + /// Gets the positive precision of a classifier which is the proportion of correctly predicted + /// positive instances among all the positive predictions (i.e., the number of positive instances + /// predicted as positive, divided by the total number of instances predicted as positive). + /// + public double PositivePrecision { get; } + + /// + /// Gets the positive recall of a classifier which is the proportion of correctly predicted + /// positive instances among all the positive instances (i.e., the number of positive instances + /// predicted as positive, divided by the total number of positive instances). + /// + public double PositiveRecall { get; private set; } + + /// + /// Gets the negative precision of a classifier which is the proportion of correctly predicted + /// negative instances among all the negative predictions (i.e., the number of negative instances + /// predicted as negative, divided by the total number of instances predicted as negative). + /// + public double NegativePrecision { get; } + + /// + /// Gets the negative recall of a classifier which is the proportion of correctly predicted + /// negative instances among all the negative instances (i.e., the number of negative instances + /// predicted as negative, divided by the total number of negative instances). + /// + public double NegativeRecall { get; } + + /// + /// Gets the F1 score of the classifier. + /// + /// + /// F1 score is the harmonic mean of precision and recall: 2 * precision * recall / (precision + recall). + /// + public double F1Score { get; } + + /// + /// Gets the area under the precision/recall curve of the classifier. + /// + /// + /// The area under the precision/recall curve is a single number summary of the information in the + /// precision/recall curve. It is increasingly used in the machine learning community, particularly + /// for imbalanced datasets where one class is observed more frequently than the other. On these + /// datasets, AUPRC can highlight performance differences that are lost with AUC. + /// + public double Auprc { get; } + + protected private static T Fetch(IExceptionContext ectx, IRow row, string name) + { + if (!row.Schema.TryGetColumnIndex(name, out int col)) + throw ectx.Except($"Could not find column '{name}'"); + T val = default; + row.GetGetter(col)(ref val); + return val; + } + + internal BinaryClassificationMetrics(IExceptionContext ectx, IRow overallResult) + { + double Fetch(string name) => Fetch(ectx, overallResult, name); + Auc = Fetch(BinaryClassifierEvaluator.Auc); + Accuracy = Fetch(BinaryClassifierEvaluator.Accuracy); + PositivePrecision = Fetch(BinaryClassifierEvaluator.PosPrecName); + PositiveRecall = Fetch(BinaryClassifierEvaluator.PosRecallName); + NegativePrecision = Fetch(BinaryClassifierEvaluator.NegPrecName); + NegativeRecall = Fetch(BinaryClassifierEvaluator.NegRecallName); + F1Score = Fetch(BinaryClassifierEvaluator.F1); + Auprc = Fetch(BinaryClassifierEvaluator.AuPrc); + } + + [BestFriend] + internal BinaryClassificationMetrics(double auc, double accuracy, double positivePrecision, double positiveRecall, + double negativePrecision, double negativeRecall, double f1Score, double auprc) + { + Auc = auc; + Accuracy = accuracy; + PositivePrecision = positivePrecision; + PositiveRecall = positiveRecall; + NegativePrecision = negativePrecision; + NegativeRecall = negativeRecall; + F1Score = f1Score; + Auprc = auprc; + } + } +} \ No newline at end of file diff --git a/src/Microsoft.ML.Data/Evaluators/Metrics/CalibratedBinaryClassificationMetrics.cs b/src/Microsoft.ML.Data/Evaluators/Metrics/CalibratedBinaryClassificationMetrics.cs new file mode 100644 index 0000000000..c4f0861224 --- /dev/null +++ b/src/Microsoft.ML.Data/Evaluators/Metrics/CalibratedBinaryClassificationMetrics.cs @@ -0,0 +1,54 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.Data; + +namespace Microsoft.ML.Data +{ + /// + /// Evaluation results for binary classifiers, including probabilistic metrics. + /// + public sealed class CalibratedBinaryClassificationMetrics : BinaryClassificationMetrics + { + /// + /// Gets the log-loss of the classifier. + /// + /// + /// The log-loss metric, is computed as follows: + /// LL = - (1/m) * sum( log(p[i])) + /// where m is the number of instances in the test set. + /// p[i] is the probability returned by the classifier if the instance belongs to class 1, + /// and 1 minus the probability returned by the classifier if the instance belongs to class 0. + /// + public double LogLoss { get; } + + /// + /// Gets the log-loss reduction (also known as relative log-loss, or reduction in information gain - RIG) + /// of the classifier. + /// + /// + /// The log-loss reduction is scaled relative to a classifier that predicts the prior for every example: + /// (LL(prior) - LL(classifier)) / LL(prior) + /// This metric can be interpreted as the advantage of the classifier over a random prediction. + /// For example, if the RIG equals 20, it can be interpreted as "the probability of a correct prediction is + /// 20% better than random guessing." + /// + public double LogLossReduction { get; } + + /// + /// Gets the test-set entropy (prior Log-Loss/instance) of the classifier. + /// + public double Entropy { get; } + + internal CalibratedBinaryClassificationMetrics(IExceptionContext ectx, IRow overallResult) + : base(ectx, overallResult) + { + double Fetch(string name) => Fetch(ectx, overallResult, name); + LogLoss = Fetch(BinaryClassifierEvaluator.LogLoss); + LogLossReduction = Fetch(BinaryClassifierEvaluator.LogLossReduction); + Entropy = Fetch(BinaryClassifierEvaluator.Entropy); + } + } +} \ No newline at end of file diff --git a/src/Microsoft.ML.Data/Evaluators/Metrics/ClusteringMetrics.cs b/src/Microsoft.ML.Data/Evaluators/Metrics/ClusteringMetrics.cs new file mode 100644 index 0000000000..fc7e87e150 --- /dev/null +++ b/src/Microsoft.ML.Data/Evaluators/Metrics/ClusteringMetrics.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.Data; + +namespace Microsoft.ML.Data +{ + /// + /// The metrics generated after evaluating the clustering predictions. + /// + public sealed class ClusteringMetrics + { + /// + /// Normalized Mutual Information + /// NMI is a measure of the mutual dependence of the variables. + /// Normalized variants work on data that already has cluster labels. + /// Its value ranged from 0 to 1, where higher numbers are better. + /// + public double Nmi { get; } + + /// + /// Average Score. For the K-Means algorithm, the 'score' is the distance from the centroid to the example. + /// The average score is, therefore, a measure of proximity of the examples to cluster centroids. + /// In other words, it's the 'cluster tightness' measure. + /// Note however, that this metric will only decrease if the number of clusters is increased, + /// and in the extreme case (where each distinct example is its own cluster) it will be equal to zero. + /// + public double AvgMinScore { get; } + + /// + /// Davies-Bouldin Index + /// DBI is a measure of the how much scatter is in the cluster and the cluster separation. + /// + public double Dbi { get; } + + internal ClusteringMetrics(IExceptionContext ectx, IRow overallResult, bool calculateDbi) + { + double Fetch(string name) => RowCursorUtils.Fetch(ectx, overallResult, name); + + Nmi = Fetch(ClusteringEvaluator.Nmi); + AvgMinScore = Fetch(ClusteringEvaluator.AvgMinScore); + + if (calculateDbi) + Dbi = Fetch(ClusteringEvaluator.Dbi); + } + } +} \ No newline at end of file diff --git a/src/Microsoft.ML.Data/Evaluators/Metrics/MultiClassClassifierMetrics.cs b/src/Microsoft.ML.Data/Evaluators/Metrics/MultiClassClassifierMetrics.cs new file mode 100644 index 0000000000..4eff184abc --- /dev/null +++ b/src/Microsoft.ML.Data/Evaluators/Metrics/MultiClassClassifierMetrics.cs @@ -0,0 +1,100 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.Data; + +namespace Microsoft.ML.Data +{ + public sealed class MultiClassClassifierMetrics + { + /// + /// Gets the micro-average accuracy of the model. + /// + /// + /// The micro-average is the fraction of instances predicted correctly. + /// + /// The micro-average metric weighs each class according to the number of instances that belong + /// to it in the dataset. + /// + public double AccuracyMicro { get; } + + /// + /// Gets the macro-average accuracy of the model. + /// + /// + /// The macro-average is computed by taking the average over all the classes of the fraction + /// of correct predictions in this class (the number of correctly predicted instances in the class, + /// divided by the total number of instances in the class). + /// + /// The macro-average metric gives the same weight to each class, no matter how many instances from + /// that class the dataset contains. + /// + public double AccuracyMacro { get; } + + /// + /// Gets the average log-loss of the classifier. + /// + /// + /// The log-loss metric, is computed as follows: + /// LL = - (1/m) * sum( log(p[i])) + /// where m is the number of instances in the test set. + /// p[i] is the probability returned by the classifier if the instance belongs to class 1, + /// and 1 minus the probability returned by the classifier if the instance belongs to class 0. + /// + public double LogLoss { get; } + + /// + /// Gets the log-loss reduction (also known as relative log-loss, or reduction in information gain - RIG) + /// of the classifier. + /// + /// + /// The log-loss reduction is scaled relative to a classifier that predicts the prior for every example: + /// (LL(prior) - LL(classifier)) / LL(prior) + /// This metric can be interpreted as the advantage of the classifier over a random prediction. + /// For example, if the RIG equals 20, it can be interpreted as "the probability of a correct prediction is + /// 20% better than random guessing". + /// + public double LogLossReduction { get; private set; } + + /// + /// If positive, this is the top-K for which the is calculated. + /// + public int TopK { get; } + + /// + /// If is positive, this is the relative number of examples where + /// the true label is one of the top k predicted labels by the predictor. + /// + public double TopKAccuracy { get; } + + /// + /// Gets the log-loss of the classifier for each class. + /// + /// + /// The log-loss metric, is computed as follows: + /// LL = - (1/m) * sum( log(p[i])) + /// where m is the number of instances in the test set. + /// p[i] is the probability returned by the classifier if the instance belongs to the class, + /// and 1 minus the probability returned by the classifier if the instance does not belong to the class. + /// + public double[] PerClassLogLoss { get; } + + internal MultiClassClassifierMetrics(IExceptionContext ectx, IRow overallResult, int topK) + { + double FetchDouble(string name) => RowCursorUtils.Fetch(ectx, overallResult, name); + AccuracyMicro = FetchDouble(MultiClassClassifierEvaluator.AccuracyMicro); + AccuracyMacro = FetchDouble(MultiClassClassifierEvaluator.AccuracyMacro); + LogLoss = FetchDouble(MultiClassClassifierEvaluator.LogLoss); + LogLossReduction = FetchDouble(MultiClassClassifierEvaluator.LogLossReduction); + TopK = topK; + if (topK > 0) + TopKAccuracy = FetchDouble(MultiClassClassifierEvaluator.TopKAccuracy); + + var perClassLogLoss = RowCursorUtils.Fetch>(ectx, overallResult, MultiClassClassifierEvaluator.PerClassLogLoss); + PerClassLogLoss = new double[perClassLogLoss.Length]; + perClassLogLoss.CopyTo(PerClassLogLoss); + } + } +} \ No newline at end of file diff --git a/src/Microsoft.ML.Data/Evaluators/Metrics/RankerMetrics.cs b/src/Microsoft.ML.Data/Evaluators/Metrics/RankerMetrics.cs new file mode 100644 index 0000000000..975d4a494c --- /dev/null +++ b/src/Microsoft.ML.Data/Evaluators/Metrics/RankerMetrics.cs @@ -0,0 +1,44 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.Data; + +namespace Microsoft.ML.Data +{ + public sealed class RankerMetrics + { + /// + /// Array of normalized discounted cumulative gains where i-th element represent NDCG@i. + /// + /// + public double[] Ndcg { get; } + + /// + ///Array of discounted cumulative gains where i-th element represent DCG@i. + /// Discounted Cumulative gain + /// is the sum of the gains, for all the instances i, normalized by the natural logarithm of the instance + 1. + /// Note that unline the Wikipedia article, ML.Net uses the natural logarithm. + /// + /// + public double[] Dcg { get; } + + private static T Fetch(IExceptionContext ectx, IRow row, string name) + { + if (!row.Schema.TryGetColumnIndex(name, out int col)) + throw ectx.Except($"Could not find column '{name}'"); + T val = default; + row.GetGetter(col)(ref val); + return val; + } + + internal RankerMetrics(IExceptionContext ectx, IRow overallResult) + { + VBuffer Fetch(string name) => Fetch>(ectx, overallResult, name); + + Dcg = Fetch(RankerEvaluator.Dcg).GetValues().ToArray(); + Ndcg = Fetch(RankerEvaluator.Ndcg).GetValues().ToArray(); + } + } +} \ No newline at end of file diff --git a/src/Microsoft.ML.Data/Evaluators/Metrics/RegressionMetrics.cs b/src/Microsoft.ML.Data/Evaluators/Metrics/RegressionMetrics.cs new file mode 100644 index 0000000000..8a9fd96b31 --- /dev/null +++ b/src/Microsoft.ML.Data/Evaluators/Metrics/RegressionMetrics.cs @@ -0,0 +1,76 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.Data; + +namespace Microsoft.ML.Data +{ + public sealed class RegressionMetrics + { + /// + /// Gets the absolute loss of the model. + /// + /// + /// The absolute loss is defined as + /// L1 = (1/m) * sum( abs( yi - y'i)) + /// where m is the number of instances in the test set. + /// y'i are the predicted labels for each instance. + /// yi are the correct labels of each instance. + /// + public double L1 { get; } + + /// + /// Gets the squared loss of the model. + /// + /// + /// The squared loss is defined as + /// L2 = (1/m) * sum(( yi - y'i)^2) + /// where m is the number of instances in the test set. + /// y'i are the predicted labels for each instance. + /// yi are the correct labels of each instance. + /// + public double L2 { get; } + + /// + /// Gets the root mean square loss (or RMS) which is the square root of the L2 loss. + /// + public double Rms { get; } + + /// + /// Gets the result of user defined loss function. + /// + /// + /// This is the average of a loss function defined by the user, + /// computed over all the instances in the test set. + /// + public double LossFn { get; } + + /// + /// Gets the R squared value of the model, which is also known as + /// the coefficient of determination​. + /// + public double RSquared { get; } + + internal RegressionMetrics(IExceptionContext ectx, IRow overallResult) + { + double Fetch(string name) => RowCursorUtils.Fetch(ectx, overallResult, name); + L1 = Fetch(RegressionEvaluator.L1); + L2 = Fetch(RegressionEvaluator.L2); + Rms = Fetch(RegressionEvaluator.Rms); + LossFn = Fetch(RegressionEvaluator.Loss); + RSquared = Fetch(RegressionEvaluator.RSquared); + } + + [BestFriend] + internal RegressionMetrics(double l1, double l2, double rms, double lossFunction, double rSquared) + { + L1 = l1; + L2 = l2; + Rms = rms; + LossFn = lossFunction; + RSquared = rSquared; + } + } +} \ No newline at end of file diff --git a/src/Microsoft.ML.Data/Evaluators/MulticlassClassifierEvaluator.cs b/src/Microsoft.ML.Data/Evaluators/MultiClassClassifierEvaluator.cs similarity index 91% rename from src/Microsoft.ML.Data/Evaluators/MulticlassClassifierEvaluator.cs rename to src/Microsoft.ML.Data/Evaluators/MultiClassClassifierEvaluator.cs index 93515e5315..273599f7f5 100644 --- a/src/Microsoft.ML.Data/Evaluators/MulticlassClassifierEvaluator.cs +++ b/src/Microsoft.ML.Data/Evaluators/MultiClassClassifierEvaluator.cs @@ -496,97 +496,6 @@ public void GetSlotNames(ref VBuffer> slotNames) } } - public sealed class Result - { - /// - /// Gets the micro-average accuracy of the model. - /// - /// - /// The micro-average is the fraction of instances predicted correctly. - /// - /// The micro-average metric weighs each class according to the number of instances that belong - /// to it in the dataset. - /// - public double AccuracyMicro { get; } - - /// - /// Gets the macro-average accuracy of the model. - /// - /// - /// The macro-average is computed by taking the average over all the classes of the fraction - /// of correct predictions in this class (the number of correctly predicted instances in the class, - /// divided by the total number of instances in the class). - /// - /// The macro-average metric gives the same weight to each class, no matter how many instances from - /// that class the dataset contains. - /// - public double AccuracyMacro { get; } - - /// - /// Gets the average log-loss of the classifier. - /// - /// - /// The log-loss metric, is computed as follows: - /// LL = - (1/m) * sum( log(p[i])) - /// where m is the number of instances in the test set. - /// p[i] is the probability returned by the classifier if the instance belongs to class 1, - /// and 1 minus the probability returned by the classifier if the instance belongs to class 0. - /// - public double LogLoss { get; } - - /// - /// Gets the log-loss reduction (also known as relative log-loss, or reduction in information gain - RIG) - /// of the classifier. - /// - /// - /// The log-loss reduction is scaled relative to a classifier that predicts the prior for every example: - /// (LL(prior) - LL(classifier)) / LL(prior) - /// This metric can be interpreted as the advantage of the classifier over a random prediction. - /// For example, if the RIG equals 20, it can be interpreted as "the probability of a correct prediction is - /// 20% better than random guessing". - /// - public double LogLossReduction { get; private set; } - - /// - /// If positive, this is the top-K for which the is calculated. - /// - public int TopK { get; } - - /// - /// If is positive, this is the relative number of examples where - /// the true label is one of the top k predicted labels by the predictor. - /// - public double TopKAccuracy { get; } - - /// - /// Gets the log-loss of the classifier for each class. - /// - /// - /// The log-loss metric, is computed as follows: - /// LL = - (1/m) * sum( log(p[i])) - /// where m is the number of instances in the test set. - /// p[i] is the probability returned by the classifier if the instance belongs to the class, - /// and 1 minus the probability returned by the classifier if the instance does not belong to the class. - /// - public double[] PerClassLogLoss { get; } - - internal Result(IExceptionContext ectx, IRow overallResult, int topK) - { - double FetchDouble(string name) => RowCursorUtils.Fetch(ectx, overallResult, name); - AccuracyMicro = FetchDouble(MultiClassClassifierEvaluator.AccuracyMicro); - AccuracyMacro = FetchDouble(MultiClassClassifierEvaluator.AccuracyMacro); - LogLoss = FetchDouble(MultiClassClassifierEvaluator.LogLoss); - LogLossReduction = FetchDouble(MultiClassClassifierEvaluator.LogLossReduction); - TopK = topK; - if (topK > 0) - TopKAccuracy = FetchDouble(MultiClassClassifierEvaluator.TopKAccuracy); - - var perClassLogLoss = RowCursorUtils.Fetch>(ectx, overallResult, MultiClassClassifierEvaluator.PerClassLogLoss); - PerClassLogLoss = new double[perClassLogLoss.Length]; - perClassLogLoss.CopyTo(PerClassLogLoss); - } - } - /// /// Evaluates scored multiclass classification data. /// @@ -595,7 +504,7 @@ internal Result(IExceptionContext ectx, IRow overallResult, int topK) /// The name of the score column in . /// The name of the predicted label column in . /// The evaluation results for these outputs. - public Result Evaluate(IDataView data, string label, string score, string predictedLabel) + public MultiClassClassifierMetrics Evaluate(IDataView data, string label, string score, string predictedLabel) { Host.CheckValue(data, nameof(data)); Host.CheckNonEmpty(label, nameof(label)); @@ -611,12 +520,12 @@ public Result Evaluate(IDataView data, string label, string score, string predic Host.Assert(resultDict.ContainsKey(MetricKinds.OverallMetrics)); var overall = resultDict[MetricKinds.OverallMetrics]; - Result result; + MultiClassClassifierMetrics result; using (var cursor = overall.GetRowCursor(i => true)) { var moved = cursor.MoveNext(); Host.Assert(moved); - result = new Result(Host, cursor, _outputTopKAcc ?? 0); + result = new MultiClassClassifierMetrics(Host, cursor, _outputTopKAcc ?? 0); moved = cursor.MoveNext(); Host.Assert(!moved); } diff --git a/src/Microsoft.ML.Data/Evaluators/RankerEvaluator.cs b/src/Microsoft.ML.Data/Evaluators/RankerEvaluator.cs index bffe37be58..a7aa5bc434 100644 --- a/src/Microsoft.ML.Data/Evaluators/RankerEvaluator.cs +++ b/src/Microsoft.ML.Data/Evaluators/RankerEvaluator.cs @@ -243,7 +243,7 @@ protected override void GetAggregatorConsolidationFuncs(Aggregator aggregator, A /// The name of the groupId column. /// The name of the predicted score column. /// The evaluation metrics for these outputs. - public Result Evaluate(IDataView data, string label, string groupId, string score) + public RankerMetrics Evaluate(IDataView data, string label, string groupId, string score) { Host.CheckValue(data, nameof(data)); Host.CheckNonEmpty(label, nameof(label)); @@ -257,12 +257,12 @@ public Result Evaluate(IDataView data, string label, string groupId, string scor Host.Assert(resultDict.ContainsKey(MetricKinds.OverallMetrics)); var overall = resultDict[MetricKinds.OverallMetrics]; - Result result; + RankerMetrics result; using (var cursor = overall.GetRowCursor(i => true)) { var moved = cursor.MoveNext(); Host.Assert(moved); - result = new Result(Host, cursor); + result = new RankerMetrics(Host, cursor); moved = cursor.MoveNext(); Host.Assert(!moved); } @@ -538,40 +538,6 @@ public void GetSlotNames(ref VBuffer> slotNames) slotNames = editor.Commit(); } } - - public sealed class Result - { - /// - /// Normalized Discounted Cumulative Gain - /// - /// - public double[] Ndcg { get; } - - /// - /// Discounted Cumulative gain - /// is the sum of the gains, for all the instances i, normalized by the natural logarithm of the instance + 1. - /// Note that unline the Wikipedia article, ML.Net uses the natural logarithm. - /// - /// - public double[] Dcg { get; } - - private static T Fetch(IExceptionContext ectx, IRow row, string name) - { - if (!row.Schema.TryGetColumnIndex(name, out int col)) - throw ectx.Except($"Could not find column '{name}'"); - T val = default; - row.GetGetter(col)(ref val); - return val; - } - - internal Result(IExceptionContext ectx, IRow overallResult) - { - VBuffer Fetch(string name) => Fetch>(ectx, overallResult, name); - - Dcg = Fetch(RankerEvaluator.Dcg).GetValues().ToArray(); - Ndcg = Fetch(RankerEvaluator.Ndcg).GetValues().ToArray(); - } - } } public sealed class RankerPerInstanceTransform : IDataTransform diff --git a/src/Microsoft.ML.Data/Evaluators/RegressionEvaluator.cs b/src/Microsoft.ML.Data/Evaluators/RegressionEvaluator.cs index df3c4a12fa..729e0bffe0 100644 --- a/src/Microsoft.ML.Data/Evaluators/RegressionEvaluator.cs +++ b/src/Microsoft.ML.Data/Evaluators/RegressionEvaluator.cs @@ -2,15 +2,15 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using Float = System.Single; using Microsoft.ML.Data; -using Microsoft.ML.Runtime; using Microsoft.ML.Runtime.CommandLine; using Microsoft.ML.Runtime.Data; using Microsoft.ML.Runtime.EntryPoints; using Microsoft.ML.Runtime.Model; -using System; +using Microsoft.ML.Runtime; using System.Collections.Generic; -using Float = System.Single; +using System; [assembly: LoadableClass(typeof(RegressionEvaluator), typeof(RegressionEvaluator), typeof(RegressionEvaluator.Arguments), typeof(SignatureEvaluator), "Regression Evaluator", RegressionEvaluator.LoadName, "Regression")] @@ -157,73 +157,6 @@ public override void AddColumn(ArrayDataViewBuilder dvBldr, string metricName, p } } - public sealed class Result - { - /// - /// Gets the absolute loss of the model. - /// - /// - /// The absolute loss is defined as - /// L1 = (1/m) * sum( abs( yi - y'i)) - /// where m is the number of instances in the test set. - /// y'i are the predicted labels for each instance. - /// yi are the correct labels of each instance. - /// - public double L1 { get; } - - /// - /// Gets the squared loss of the model. - /// - /// - /// The squared loss is defined as - /// L2 = (1/m) * sum(( yi - y'i)^2) - /// where m is the number of instances in the test set. - /// y'i are the predicted labels for each instance. - /// yi are the correct labels of each instance. - /// - public double L2 { get; } - - /// - /// Gets the root mean square loss (or RMS) which is the square root of the L2 loss. - /// - public double Rms { get; } - - /// - /// Gets the user defined loss function. - /// - /// - /// This is the average of a loss function defined by the user, - /// computed over all the instances in the test set. - /// - public double LossFn { get; } - - /// - /// Gets the R squared value of the model, which is also known as - /// the coefficient of determination​. - /// - public double RSquared { get; } - - internal Result(IExceptionContext ectx, IRow overallResult) - { - double Fetch(string name) => RowCursorUtils.Fetch(ectx, overallResult, name); - L1 = Fetch(RegressionEvaluator.L1); - L2 = Fetch(RegressionEvaluator.L2); - Rms = Fetch(RegressionEvaluator.Rms); - LossFn = Fetch(RegressionEvaluator.Loss); - RSquared = Fetch(RegressionEvaluator.RSquared); - } - - [BestFriend] - internal Result(double l1, double l2, double rms, double lossFunction, double rSquared) - { - L1 = l1; - L2 = l2; - Rms = rms; - LossFn = lossFunction; - RSquared = rSquared; - } - } - /// /// Evaluates scored regression data. /// @@ -231,7 +164,7 @@ internal Result(double l1, double l2, double rms, double lossFunction, double rS /// The name of the label column. /// The name of the predicted score column. /// The evaluation metrics for these outputs. - public Result Evaluate(IDataView data, string label, string score) + public RegressionMetrics Evaluate(IDataView data, string label, string score) { Host.CheckValue(data, nameof(data)); Host.CheckNonEmpty(label, nameof(label)); @@ -244,12 +177,12 @@ public Result Evaluate(IDataView data, string label, string score) Host.Assert(resultDict.ContainsKey(MetricKinds.OverallMetrics)); var overall = resultDict[MetricKinds.OverallMetrics]; - Result result; + RegressionMetrics result; using (var cursor = overall.GetRowCursor(i => true)) { var moved = cursor.MoveNext(); Host.Assert(moved); - result = new Result(Host, cursor); + result = new RegressionMetrics(Host, cursor); moved = cursor.MoveNext(); Host.Assert(!moved); } diff --git a/src/Microsoft.ML.Data/TrainContext.cs b/src/Microsoft.ML.Data/TrainContext.cs index 9418efae89..13eda65945 100644 --- a/src/Microsoft.ML.Data/TrainContext.cs +++ b/src/Microsoft.ML.Data/TrainContext.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using Microsoft.ML.Core.Data; +using Microsoft.ML.Data; using Microsoft.ML.Runtime; using Microsoft.ML.Runtime.Data; using Microsoft.ML.Transforms; @@ -207,7 +208,7 @@ internal BinaryClassificationTrainers(BinaryClassificationContext ctx) /// The name of the probability column in , the calibrated version of . /// The name of the predicted label column in . /// The evaluation results for these calibrated outputs. - public BinaryClassifierEvaluator.CalibratedResult Evaluate(IDataView data, string label = DefaultColumnNames.Label, string score = DefaultColumnNames.Score, + public CalibratedBinaryClassificationMetrics Evaluate(IDataView data, string label = DefaultColumnNames.Label, string score = DefaultColumnNames.Score, string probability = DefaultColumnNames.Probability, string predictedLabel = DefaultColumnNames.PredictedLabel) { Host.CheckValue(data, nameof(data)); @@ -228,7 +229,7 @@ public BinaryClassifierEvaluator.CalibratedResult Evaluate(IDataView data, strin /// The name of the score column in . /// The name of the predicted label column in . /// The evaluation results for these uncalibrated outputs. - public BinaryClassifierEvaluator.Result EvaluateNonCalibrated(IDataView data, string label = DefaultColumnNames.Label, string score = DefaultColumnNames.Score, + public BinaryClassificationMetrics EvaluateNonCalibrated(IDataView data, string label = DefaultColumnNames.Label, string score = DefaultColumnNames.Score, string predictedLabel = DefaultColumnNames.PredictedLabel) { Host.CheckValue(data, nameof(data)); @@ -254,7 +255,7 @@ public BinaryClassifierEvaluator.Result EvaluateNonCalibrated(IDataView data, st /// they are guaranteed to appear in the same subset (train or test). Use this to make sure there is no label leakage from /// train to the test set. /// Per-fold results: metrics, models, scored datasets. - public (BinaryClassifierEvaluator.Result metrics, ITransformer model, IDataView scoredTestData)[] CrossValidateNonCalibrated( + public (BinaryClassificationMetrics metrics, ITransformer model, IDataView scoredTestData)[] CrossValidateNonCalibrated( IDataView data, IEstimator estimator, int numFolds = 5, string labelColumn = DefaultColumnNames.Label, string stratificationColumn = null) { Host.CheckNonEmpty(labelColumn, nameof(labelColumn)); @@ -276,7 +277,7 @@ public BinaryClassifierEvaluator.Result EvaluateNonCalibrated(IDataView data, st /// they are guaranteed to appear in the same subset (train or test). Use this to make sure there is no label leakage from /// train to the test set. /// Per-fold results: metrics, models, scored datasets. - public (BinaryClassifierEvaluator.CalibratedResult metrics, ITransformer model, IDataView scoredTestData)[] CrossValidate( + public (CalibratedBinaryClassificationMetrics metrics, ITransformer model, IDataView scoredTestData)[] CrossValidate( IDataView data, IEstimator estimator, int numFolds = 5, string labelColumn = DefaultColumnNames.Label, string stratificationColumn = null) { Host.CheckNonEmpty(labelColumn, nameof(labelColumn)); @@ -318,11 +319,11 @@ internal ClusteringTrainers(ClusteringContext ctx) /// The scored data. /// The name of the score column in . /// The name of the optional label column in . - /// If present, the metric will be computed. + /// If present, the metric will be computed. /// The name of the optional features column in . - /// If present, the metric will be computed. + /// If present, the metric will be computed. /// The evaluation result. - public ClusteringEvaluator.Result Evaluate(IDataView data, + public ClusteringMetrics Evaluate(IDataView data, string label = null, string score = DefaultColumnNames.Score, string features = null ) @@ -355,7 +356,7 @@ public ClusteringEvaluator.Result Evaluate(IDataView data, /// they are guaranteed to appear in the same subset (train or test). Use this to make sure there is no label leakage from /// train to the test set. /// Per-fold results: metrics, models, scored datasets. - public (ClusteringEvaluator.Result metrics, ITransformer model, IDataView scoredTestData)[] CrossValidate( + public (ClusteringMetrics metrics, ITransformer model, IDataView scoredTestData)[] CrossValidate( IDataView data, IEstimator estimator, int numFolds = 5, string labelColumn = null, string featuresColumn = null, string stratificationColumn = null) { var result = CrossValidateTrain(data, estimator, numFolds, stratificationColumn); @@ -394,11 +395,11 @@ internal MulticlassClassificationTrainers(MulticlassClassificationContext ctx) /// The name of the label column in . /// The name of the score column in . /// The name of the predicted label column in . - /// If given a positive value, the will be filled with + /// If given a positive value, the will be filled with /// the top-K accuracy, that is, the accuracy assuming we consider an example with the correct class within /// the top-K values as being stored "correctly." /// The evaluation results for these calibrated outputs. - public MultiClassClassifierEvaluator.Result Evaluate(IDataView data, string label = DefaultColumnNames.Label, string score = DefaultColumnNames.Score, + public MultiClassClassifierMetrics Evaluate(IDataView data, string label = DefaultColumnNames.Label, string score = DefaultColumnNames.Score, string predictedLabel = DefaultColumnNames.PredictedLabel, int topK = 0) { Host.CheckValue(data, nameof(data)); @@ -427,7 +428,7 @@ public MultiClassClassifierEvaluator.Result Evaluate(IDataView data, string labe /// they are guaranteed to appear in the same subset (train or test). Use this to make sure there is no label leakage from /// train to the test set. /// Per-fold results: metrics, models, scored datasets. - public (MultiClassClassifierEvaluator.Result metrics, ITransformer model, IDataView scoredTestData)[] CrossValidate( + public (MultiClassClassifierMetrics metrics, ITransformer model, IDataView scoredTestData)[] CrossValidate( IDataView data, IEstimator estimator, int numFolds = 5, string labelColumn = DefaultColumnNames.Label, string stratificationColumn = null) { Host.CheckNonEmpty(labelColumn, nameof(labelColumn)); @@ -467,7 +468,7 @@ internal RegressionTrainers(RegressionContext ctx) /// The name of the label column in . /// The name of the score column in . /// The evaluation results for these calibrated outputs. - public RegressionEvaluator.Result Evaluate(IDataView data, string label = DefaultColumnNames.Label, string score = DefaultColumnNames.Score) + public RegressionMetrics Evaluate(IDataView data, string label = DefaultColumnNames.Label, string score = DefaultColumnNames.Score) { Host.CheckValue(data, nameof(data)); Host.CheckNonEmpty(label, nameof(label)); @@ -491,7 +492,7 @@ public RegressionEvaluator.Result Evaluate(IDataView data, string label = Defaul /// they are guaranteed to appear in the same subset (train or test). Use this to make sure there is no label leakage from /// train to the test set. /// Per-fold results: metrics, models, scored datasets. - public (RegressionEvaluator.Result metrics, ITransformer model, IDataView scoredTestData)[] CrossValidate( + public (RegressionMetrics metrics, ITransformer model, IDataView scoredTestData)[] CrossValidate( IDataView data, IEstimator estimator, int numFolds = 5, string labelColumn = DefaultColumnNames.Label, string stratificationColumn = null) { Host.CheckNonEmpty(labelColumn, nameof(labelColumn)); @@ -532,7 +533,7 @@ internal RankingTrainers(RankingContext ctx) /// The name of the groupId column in . /// The name of the score column in . /// The evaluation results for these calibrated outputs. - public RankerEvaluator.Result Evaluate(IDataView data, string label, string groupId, string score = DefaultColumnNames.Score) + public RankerMetrics Evaluate(IDataView data, string label, string groupId, string score = DefaultColumnNames.Score) { Host.CheckValue(data, nameof(data)); Host.CheckNonEmpty(label, nameof(label)); diff --git a/src/Microsoft.ML.Data/Training/TrainingStaticExtensions.cs b/src/Microsoft.ML.Data/Training/TrainingStaticExtensions.cs index 8cfda0485e..1107a24a83 100644 --- a/src/Microsoft.ML.Data/Training/TrainingStaticExtensions.cs +++ b/src/Microsoft.ML.Data/Training/TrainingStaticExtensions.cs @@ -3,14 +3,12 @@ // See the LICENSE file in the project root for more information. using Microsoft.ML.Core.Data; -using Microsoft.ML.StaticPipe; -using Microsoft.ML.StaticPipe.Runtime; +using Microsoft.ML.Data; using Microsoft.ML.Runtime; -using Microsoft.ML.Runtime.Data; -using System; -using System.Collections.Generic; +using Microsoft.ML.StaticPipe.Runtime; +using Microsoft.ML.StaticPipe; using System.Linq; -using System.Text; +using System; namespace Microsoft.ML { @@ -73,7 +71,7 @@ public static (DataView trainSet, DataView testSet) TrainTestSplit(this /// they are guaranteed to appear in the same subset (train or test). Use this to make sure there is no label leakage from /// train to the test set. /// Per-fold results: metrics, models, scored datasets. - public static (RegressionEvaluator.Result metrics, Transformer model, DataView scoredTestData)[] CrossValidate( + public static (RegressionMetrics metrics, Transformer model, DataView scoredTestData)[] CrossValidate( this RegressionContext context, DataView data, Estimator estimator, @@ -129,7 +127,7 @@ public static (RegressionEvaluator.Result metrics, Transformer /// Per-fold results: metrics, models, scored datasets. - public static (MultiClassClassifierEvaluator.Result metrics, Transformer model, DataView scoredTestData)[] CrossValidate( + public static (MultiClassClassifierMetrics metrics, Transformer model, DataView scoredTestData)[] CrossValidate( this MulticlassClassificationContext context, DataView data, Estimator estimator, @@ -185,7 +183,7 @@ public static (MultiClassClassifierEvaluator.Result metrics, Transformer /// Per-fold results: metrics, models, scored datasets. - public static (BinaryClassifierEvaluator.Result metrics, Transformer model, DataView scoredTestData)[] CrossValidateNonCalibrated( + public static (BinaryClassificationMetrics metrics, Transformer model, DataView scoredTestData)[] CrossValidateNonCalibrated( this BinaryClassificationContext context, DataView data, Estimator estimator, @@ -241,7 +239,7 @@ public static (BinaryClassifierEvaluator.Result metrics, Transformer /// Per-fold results: metrics, models, scored datasets. - public static (BinaryClassifierEvaluator.CalibratedResult metrics, Transformer model, DataView scoredTestData)[] CrossValidate( + public static (CalibratedBinaryClassificationMetrics metrics, Transformer model, DataView scoredTestData)[] CrossValidate( this BinaryClassificationContext context, DataView data, Estimator estimator, diff --git a/src/Microsoft.ML.Recommender/RecommenderCatalog.cs b/src/Microsoft.ML.Recommender/RecommenderCatalog.cs index 1100b66013..2be553736f 100644 --- a/src/Microsoft.ML.Recommender/RecommenderCatalog.cs +++ b/src/Microsoft.ML.Recommender/RecommenderCatalog.cs @@ -3,11 +3,12 @@ // See the LICENSE file in the project root for more information. using Microsoft.ML.Core.Data; -using Microsoft.ML.Runtime; +using Microsoft.ML.Data; using Microsoft.ML.Runtime.Data; +using Microsoft.ML.Runtime; using Microsoft.ML.Trainers; -using System; using System.Linq; +using System; namespace Microsoft.ML { @@ -71,7 +72,7 @@ public MatrixFactorizationTrainer MatrixFactorization( /// The name of the label column in . /// The name of the score column in . /// The evaluation results for these calibrated outputs. - public RegressionEvaluator.Result Evaluate(IDataView data, string label = DefaultColumnNames.Label, string score = DefaultColumnNames.Score) + public RegressionMetrics Evaluate(IDataView data, string label = DefaultColumnNames.Label, string score = DefaultColumnNames.Score) { Host.CheckValue(data, nameof(data)); Host.CheckNonEmpty(label, nameof(label)); @@ -95,7 +96,7 @@ public RegressionEvaluator.Result Evaluate(IDataView data, string label = Defaul /// they are guaranteed to appear in the same subset (train or test). Use this to make sure there is no label leakage from /// train to the test set. /// Per-fold results: metrics, models, scored datasets. - public (RegressionEvaluator.Result metrics, ITransformer model, IDataView scoredTestData)[] CrossValidate( + public (RegressionMetrics metrics, ITransformer model, IDataView scoredTestData)[] CrossValidate( IDataView data, IEstimator estimator, int numFolds = 5, string labelColumn = DefaultColumnNames.Label, string stratificationColumn = null) { Host.CheckNonEmpty(labelColumn, nameof(labelColumn)); diff --git a/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs b/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs index 5b726f2d41..07bfa4ebfc 100644 --- a/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs +++ b/src/Microsoft.ML.Transforms/PermutationFeatureImportanceExtensions.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using Microsoft.ML.Data; using Microsoft.ML.Runtime; using Microsoft.ML.Runtime.Data; using Microsoft.ML.Transforms; @@ -51,7 +52,7 @@ public static class PermutationFeatureImportanceExtensions /// Use features weight to pre-filter features. /// Limit the number of examples to evaluate on. null means examples (up to ~ 2 bln) from input will be used. /// Array of per-feature 'contributions' to the score. - public static ImmutableArray + public static ImmutableArray PermutationFeatureImportance( this RegressionContext ctx, IPredictionTransformer model, @@ -61,7 +62,7 @@ public static ImmutableArray bool useFeatureWeightFilter = false, int? topExamples = null) { - return PermutationFeatureImportance.GetImportanceMetricsMatrix( + return PermutationFeatureImportance.GetImportanceMetricsMatrix( CatalogUtils.GetEnvironment(ctx), model, data, @@ -72,10 +73,10 @@ public static ImmutableArray topExamples); } - private static RegressionEvaluator.Result RegressionDelta( - RegressionEvaluator.Result a, RegressionEvaluator.Result b) + private static RegressionMetrics RegressionDelta( + RegressionMetrics a, RegressionMetrics b) { - return new RegressionEvaluator.Result( + return new RegressionMetrics( l1: a.L1 - b.L1, l2: a.L2 - b.L2, rms: a.Rms - b.Rms, @@ -96,7 +97,7 @@ private static RegressionEvaluator.Result RegressionDelta( /// Use features weight to pre-filter features. /// Limit the number of examples to evaluate on. null means examples (up to ~ 2 bln) from input will be used. /// Array of per-feature 'contributions' to the score. - public static ImmutableArray + public static ImmutableArray PermutationFeatureImportance( this BinaryClassificationContext ctx, IPredictionTransformer model, @@ -106,7 +107,7 @@ public static ImmutableArray bool useFeatureWeightFilter = false, int? topExamples = null) { - return PermutationFeatureImportance.GetImportanceMetricsMatrix( + return PermutationFeatureImportance.GetImportanceMetricsMatrix( CatalogUtils.GetEnvironment(ctx), model, data, @@ -117,10 +118,10 @@ public static ImmutableArray topExamples); } - private static BinaryClassifierEvaluator.Result BinaryClassifierDelta( - BinaryClassifierEvaluator.Result a, BinaryClassifierEvaluator.Result b) + private static BinaryClassificationMetrics BinaryClassifierDelta( + BinaryClassificationMetrics a, BinaryClassificationMetrics b) { - return new BinaryClassifierEvaluator.Result( + return new BinaryClassificationMetrics( auc: a.Auc - b.Auc, accuracy: a.Accuracy - b.Accuracy, positivePrecision: a.PositivePrecision - b.PositivePrecision, diff --git a/src/Native/MatrixFactorizationNative/libmf b/src/Native/MatrixFactorizationNative/libmf index f92a18161b..53a91e7e8c 160000 --- a/src/Native/MatrixFactorizationNative/libmf +++ b/src/Native/MatrixFactorizationNative/libmf @@ -1 +1 @@ -Subproject commit f92a18161b6824fda4c4ab698a69d299a836841a +Subproject commit 53a91e7e8c88463e97acfbbafb7134a6030860b3 diff --git a/test/Microsoft.ML.Tests/Scenarios/SentimentPredictionTests.cs b/test/Microsoft.ML.Tests/Scenarios/SentimentPredictionTests.cs index 226257fc48..a8be68d31a 100644 --- a/test/Microsoft.ML.Tests/Scenarios/SentimentPredictionTests.cs +++ b/test/Microsoft.ML.Tests/Scenarios/SentimentPredictionTests.cs @@ -189,7 +189,7 @@ public void CrossValidateSentimentModelTest() Assert.True(predictions.ElementAt(1).Sentiment); } - private void ValidateBinaryMetricsSymSGD(BinaryClassificationMetrics metrics) + private void ValidateBinaryMetricsSymSGD(Microsoft.ML.Legacy.Models.BinaryClassificationMetrics metrics) { Assert.Equal(.8889, metrics.Accuracy, 4); @@ -222,7 +222,7 @@ private void ValidateBinaryMetricsSymSGD(BinaryClassificationMetrics metrics) } - private void ValidateBinaryMetricsLightGBM(BinaryClassificationMetrics metrics) + private void ValidateBinaryMetricsLightGBM(Microsoft.ML.Legacy.Models.BinaryClassificationMetrics metrics) { Assert.Equal(0.61111111111111116, metrics.Accuracy, 4); @@ -255,7 +255,7 @@ private void ValidateBinaryMetricsLightGBM(BinaryClassificationMetrics metrics) } - private void ValidateBinaryMetrics(BinaryClassificationMetrics metrics) + private void ValidateBinaryMetrics(Microsoft.ML.Legacy.Models.BinaryClassificationMetrics metrics) { Assert.Equal(0.6111, metrics.Accuracy, 4); diff --git a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/IrisPlantClassificationTests.cs b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/IrisPlantClassificationTests.cs index 9d7ac5cf81..01b3d75d13 100644 --- a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/IrisPlantClassificationTests.cs +++ b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/IrisPlantClassificationTests.cs @@ -99,7 +99,7 @@ private void ComparePredictions(PredictionFunction mod Assert.Equal(0, prediction.PredictedLabels[2], 2); } - private void CompareMatrics(MultiClassClassifierEvaluator.Result metrics) + private void CompareMatrics(MultiClassClassifierMetrics metrics) { Assert.Equal(.98, metrics.AccuracyMacro); Assert.Equal(.98, metrics.AccuracyMicro, 2); diff --git a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/SentimentPredictionTests.cs b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/SentimentPredictionTests.cs index 0d298a51b5..b04a360168 100644 --- a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/SentimentPredictionTests.cs +++ b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/SentimentPredictionTests.cs @@ -152,7 +152,7 @@ public void TrainAndPredictSentimentModelWithDirectionInstantiationTestWithWordE Assert.Equal(1.0, (double)summary[0].Value, 1); } - private BinaryClassificationMetrics EvaluateBinary(IHostEnvironment env, IDataView scoredData) + private Microsoft.ML.Legacy.Models.BinaryClassificationMetrics EvaluateBinary(IHostEnvironment env, IDataView scoredData) { var dataEval = new RoleMappedData(scoredData, label: "Label", feature: "Features", opt: true); @@ -163,7 +163,8 @@ private BinaryClassificationMetrics EvaluateBinary(IHostEnvironment env, IDataVi var evaluator = new BinaryClassifierMamlEvaluator(env, new BinaryClassifierMamlEvaluator.Arguments()); var metricsDic = evaluator.Evaluate(dataEval); - return BinaryClassificationMetrics.FromMetrics(env, metricsDic["OverallMetrics"], metricsDic["ConfusionMatrix"])[0]; + return Microsoft.ML.Legacy.Models.BinaryClassificationMetrics + .FromMetrics(env, metricsDic["OverallMetrics"], metricsDic["ConfusionMatrix"])[0]; } } #pragma warning restore 612