Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update PFI tests so they configure sensitive learners to use a single thread. #1926

Merged
merged 1 commit into from
Dec 19, 2018
Merged

Update PFI tests so they configure sensitive learners to use a single thread. #1926

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
168 changes: 84 additions & 84 deletions test/Microsoft.ML.Tests/PermutationFeatureImportanceTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,18 +40,18 @@ public void TestPfiRegressionOnDenseFeatures()
// X4Rand: 3

// For the following metrics lower is better, so maximum delta means more important feature, and vice versa
Assert.True(MinDeltaIndex(pfi, m => m.L1) == 3);
Assert.True(MaxDeltaIndex(pfi, m => m.L1) == 1);
Assert.Equal(3, MinDeltaIndex(pfi, m => m.L1));
Assert.Equal(1, MaxDeltaIndex(pfi, m => m.L1));

Assert.True(MinDeltaIndex(pfi, m => m.L2) == 3);
Assert.True(MaxDeltaIndex(pfi, m => m.L2) == 1);
Assert.Equal(3, MinDeltaIndex(pfi, m => m.L2));
Assert.Equal(1, MaxDeltaIndex(pfi, m => m.L2));

Assert.True(MinDeltaIndex(pfi, m => m.Rms) == 3);
Assert.True(MaxDeltaIndex(pfi, m => m.Rms) == 1);
Assert.Equal(3, MinDeltaIndex(pfi, m => m.Rms));
Assert.Equal(1, MaxDeltaIndex(pfi, m => m.Rms));

// For the following metrics higher is better, so minimum delta means more important feature, and vice versa
Assert.True(MaxDeltaIndex(pfi, m => m.RSquared) == 3);
Assert.True(MinDeltaIndex(pfi, m => m.RSquared) == 1);
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.RSquared));
Assert.Equal(1, MinDeltaIndex(pfi, m => m.RSquared));

Done();
}
Expand All @@ -76,18 +76,18 @@ public void TestPfiRegressionOnSparseFeatures()

// Permuted X2VBuffer-Slot-1 lot (f2) should have min impact on SGD metrics, X3Important -- max impact.
// For the following metrics lower is better, so maximum delta means more important feature, and vice versa
Assert.True(MinDeltaIndex(results, m => m.L1) == 2);
Assert.True(MaxDeltaIndex(results, m => m.L1) == 5);
Assert.Equal(2, MinDeltaIndex(results, m => m.L1));
Assert.Equal(5, MaxDeltaIndex(results, m => m.L1));

Assert.True(MinDeltaIndex(results, m => m.L2) == 2);
Assert.True(MaxDeltaIndex(results, m => m.L2) == 5);
Assert.Equal(2, MinDeltaIndex(results, m => m.L2));
Assert.Equal(5, MaxDeltaIndex(results, m => m.L2));

Assert.True(MinDeltaIndex(results, m => m.Rms) == 2);
Assert.True(MaxDeltaIndex(results, m => m.Rms) == 5);
Assert.Equal(2, MinDeltaIndex(results, m => m.Rms));
Assert.Equal(5, MaxDeltaIndex(results, m => m.Rms));

// For the following metrics higher is better, so minimum delta means more important feature, and vice versa
Assert.True(MaxDeltaIndex(results, m => m.RSquared) == 2);
Assert.True(MinDeltaIndex(results, m => m.RSquared) == 5);
Assert.Equal(2, MaxDeltaIndex(results, m => m.RSquared));
Assert.Equal(5, MinDeltaIndex(results, m => m.RSquared));
}

#endregion
Expand All @@ -100,7 +100,7 @@ public void TestPfiRegressionOnSparseFeatures()
public void TestPfiBinaryClassificationOnDenseFeatures()
{
var data = GetDenseDataset(TaskType.BinaryClassification);
var model = ML.BinaryClassification.Trainers.LogisticRegression().Fit(data);
var model = ML.BinaryClassification.Trainers.LogisticRegression(advancedSettings: args => args.NumThreads = 1).Fit(data);
var pfi = ML.BinaryClassification.PermutationFeatureImportance(model, data);

// Pfi Indices:
Expand All @@ -110,22 +110,22 @@ public void TestPfiBinaryClassificationOnDenseFeatures()
// X4Rand: 3

// For the following metrics higher is better, so minimum delta means more important feature, and vice versa
Assert.True(MaxDeltaIndex(pfi, m => m.Auc) == 3);
Assert.True(MinDeltaIndex(pfi, m => m.Auc) == 1);
Assert.True(MaxDeltaIndex(pfi, m => m.Accuracy) == 3);
Assert.True(MinDeltaIndex(pfi, m => m.Accuracy) == 1);
Assert.True(MaxDeltaIndex(pfi, m => m.PositivePrecision) == 3);
Assert.True(MinDeltaIndex(pfi, m => m.PositivePrecision) == 1);
Assert.True(MaxDeltaIndex(pfi, m => m.PositiveRecall) == 3);
Assert.True(MinDeltaIndex(pfi, m => m.PositiveRecall) == 1);
Assert.True(MaxDeltaIndex(pfi, m => m.NegativePrecision) == 3);
Assert.True(MinDeltaIndex(pfi, m => m.NegativePrecision) == 1);
Assert.True(MaxDeltaIndex(pfi, m => m.NegativeRecall) == 3);
Assert.True(MinDeltaIndex(pfi, m => m.NegativeRecall) == 1);
Assert.True(MaxDeltaIndex(pfi, m => m.F1Score) == 3);
Assert.True(MinDeltaIndex(pfi, m => m.F1Score) == 1);
Assert.True(MaxDeltaIndex(pfi, m => m.Auprc) == 3);
Assert.True(MinDeltaIndex(pfi, m => m.Auprc) == 1);
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.Auc));
Assert.Equal(1, MinDeltaIndex(pfi, m => m.Auc));
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.Accuracy));
Assert.Equal(1, MinDeltaIndex(pfi, m => m.Accuracy));
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.PositivePrecision));
Assert.Equal(1, MinDeltaIndex(pfi, m => m.PositivePrecision));
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.PositiveRecall));
Assert.Equal(1, MinDeltaIndex(pfi, m => m.PositiveRecall));
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.NegativePrecision));
Assert.Equal(1, MinDeltaIndex(pfi, m => m.NegativePrecision));
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.NegativeRecall));
Assert.Equal(1, MinDeltaIndex(pfi, m => m.NegativeRecall));
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.F1Score));
Assert.Equal(1, MinDeltaIndex(pfi, m => m.F1Score));
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.Auprc));
Assert.Equal(1, MinDeltaIndex(pfi, m => m.Auprc));

Done();
}
Expand All @@ -137,7 +137,7 @@ public void TestPfiBinaryClassificationOnDenseFeatures()
public void TestPfiBinaryClassificationOnSparseFeatures()
{
var data = GetSparseDataset(TaskType.BinaryClassification);
var model = ML.BinaryClassification.Trainers.LogisticRegression().Fit(data);
var model = ML.BinaryClassification.Trainers.LogisticRegression(advancedSettings: args => args.NumThreads = 1).Fit(data);
var pfi = ML.BinaryClassification.PermutationFeatureImportance(model, data);

// Pfi Indices:
Expand All @@ -149,22 +149,22 @@ public void TestPfiBinaryClassificationOnSparseFeatures()
// X3Important: 5

// For the following metrics higher is better, so minimum delta means more important feature, and vice versa
Assert.True(MaxDeltaIndex(pfi, m => m.Auc) == 2);
Assert.True(MinDeltaIndex(pfi, m => m.Auc) == 5);
Assert.True(MaxDeltaIndex(pfi, m => m.Accuracy) == 2);
Assert.True(MinDeltaIndex(pfi, m => m.Accuracy) == 5);
Assert.True(MaxDeltaIndex(pfi, m => m.PositivePrecision) == 2);
Assert.True(MinDeltaIndex(pfi, m => m.PositivePrecision) == 5);
Assert.True(MaxDeltaIndex(pfi, m => m.PositiveRecall) == 2);
Assert.True(MinDeltaIndex(pfi, m => m.PositiveRecall) == 5);
Assert.True(MaxDeltaIndex(pfi, m => m.NegativePrecision) == 2);
Assert.True(MinDeltaIndex(pfi, m => m.NegativePrecision) == 5);
Assert.True(MaxDeltaIndex(pfi, m => m.NegativeRecall) == 2);
Assert.True(MinDeltaIndex(pfi, m => m.NegativeRecall) == 5);
Assert.True(MaxDeltaIndex(pfi, m => m.F1Score) == 2);
Assert.True(MinDeltaIndex(pfi, m => m.F1Score) == 5);
Assert.True(MaxDeltaIndex(pfi, m => m.Auprc) == 2);
Assert.True(MinDeltaIndex(pfi, m => m.Auprc) == 5);
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.Auc));
Assert.Equal(5, MinDeltaIndex(pfi, m => m.Auc));
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.Accuracy));
Assert.Equal(5, MinDeltaIndex(pfi, m => m.Accuracy));
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.PositivePrecision));
Assert.Equal(5, MinDeltaIndex(pfi, m => m.PositivePrecision));
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.PositiveRecall));
Assert.Equal(5, MinDeltaIndex(pfi, m => m.PositiveRecall));
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.NegativePrecision));
Assert.Equal(5, MinDeltaIndex(pfi, m => m.NegativePrecision));
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.NegativeRecall));
Assert.Equal(5, MinDeltaIndex(pfi, m => m.NegativeRecall));
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.F1Score));
Assert.Equal(5, MinDeltaIndex(pfi, m => m.F1Score));
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.Auprc));
Assert.Equal(5, MinDeltaIndex(pfi, m => m.Auprc));

Done();
}
Expand All @@ -188,21 +188,21 @@ public void TestPfiMulticlassClassificationOnDenseFeatures()
// X4Rand: 3

// For the following metrics higher is better, so minimum delta means more important feature, and vice versa
Assert.True(MaxDeltaIndex(pfi, m => m.AccuracyMicro) == 3);
Assert.True(MinDeltaIndex(pfi, m => m.AccuracyMicro) == 1);
Assert.True(MaxDeltaIndex(pfi, m => m.AccuracyMacro) == 3);
Assert.True(MinDeltaIndex(pfi, m => m.AccuracyMacro) == 1);
Assert.True(MaxDeltaIndex(pfi, m => m.LogLossReduction) == 3);
Assert.True(MinDeltaIndex(pfi, m => m.LogLossReduction) == 1);
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.AccuracyMicro));
Assert.Equal(1, MinDeltaIndex(pfi, m => m.AccuracyMicro));
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.AccuracyMacro));
Assert.Equal(1, MinDeltaIndex(pfi, m => m.AccuracyMacro));
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.LogLossReduction));
Assert.Equal(1, MinDeltaIndex(pfi, m => m.LogLossReduction));

// For the following metrics-delta lower is better, so maximum delta means more important feature, and vice versa
// Because they are _negative_, the difference will be positive for worse classifiers.
Assert.True(MaxDeltaIndex(pfi, m => m.LogLoss) == 1);
Assert.True(MinDeltaIndex(pfi, m => m.LogLoss) == 3);
Assert.Equal(1, MaxDeltaIndex(pfi, m => m.LogLoss));
Assert.Equal(3, MinDeltaIndex(pfi, m => m.LogLoss));
for (int i = 0; i < pfi[0].PerClassLogLoss.Length; i++)
{
Assert.True(MaxDeltaIndex(pfi, m => m.PerClassLogLoss[i]) == 1);
Assert.True(MinDeltaIndex(pfi, m => m.PerClassLogLoss[i]) == 3);
Assert.Equal(1, MaxDeltaIndex(pfi, m => m.PerClassLogLoss[i]));
Assert.Equal(3, MinDeltaIndex(pfi, m => m.PerClassLogLoss[i]));
}

Done();
Expand All @@ -227,21 +227,21 @@ public void TestPfiMulticlassClassificationOnSparseFeatures()
// X3Important: 5 // Most important

// For the following metrics higher is better, so minimum delta means more important feature, and vice versa
Assert.True(MaxDeltaIndex(pfi, m => m.AccuracyMicro) == 2);
Assert.True(MinDeltaIndex(pfi, m => m.AccuracyMicro) == 5);
Assert.True(MaxDeltaIndex(pfi, m => m.AccuracyMacro) == 2);
Assert.True(MinDeltaIndex(pfi, m => m.AccuracyMacro) == 5);
Assert.True(MaxDeltaIndex(pfi, m => m.LogLossReduction) == 2);
Assert.True(MinDeltaIndex(pfi, m => m.LogLossReduction) == 5);
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.AccuracyMicro));
Assert.Equal(5, MinDeltaIndex(pfi, m => m.AccuracyMicro));
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.AccuracyMacro));
Assert.Equal(5, MinDeltaIndex(pfi, m => m.AccuracyMacro));
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.LogLossReduction));
Assert.Equal(5, MinDeltaIndex(pfi, m => m.LogLossReduction));

// For the following metrics-delta lower is better, so maximum delta means more important feature, and vice versa
// Because they are negative metrics, the _difference_ will be positive for worse classifiers.
Assert.True(MaxDeltaIndex(pfi, m => m.LogLoss) == 5);
Assert.True(MinDeltaIndex(pfi, m => m.LogLoss) == 2);
Assert.Equal(5, MaxDeltaIndex(pfi, m => m.LogLoss));
Assert.Equal(2, MinDeltaIndex(pfi, m => m.LogLoss));
for (int i = 0; i < pfi[0].PerClassLogLoss.Length; i++)
{
Assert.True(MaxDeltaIndex(pfi, m => m.PerClassLogLoss[i]) == 5);
Assert.True(MinDeltaIndex(pfi, m => m.PerClassLogLoss[i]) == 2);
Assert.Equal(5, MaxDeltaIndex(pfi, m => m.PerClassLogLoss[i]));
Assert.Equal(2, MinDeltaIndex(pfi, m => m.PerClassLogLoss[i]));
}

Done();
Expand All @@ -268,13 +268,13 @@ public void TestPfiRankingOnDenseFeatures()
// For the following metrics higher is better, so minimum delta means more important feature, and vice versa
for (int i = 0; i < pfi[0].Dcg.Length; i++)
{
Assert.True(MaxDeltaIndex(pfi, m => m.Dcg[i]) == 0);
Assert.True(MinDeltaIndex(pfi, m => m.Dcg[i]) == 1);
Assert.Equal(0, MaxDeltaIndex(pfi, m => m.Dcg[i]));
Assert.Equal(1, MinDeltaIndex(pfi, m => m.Dcg[i]));
}
for (int i = 0; i < pfi[0].Ndcg.Length; i++)
{
Assert.True(MaxDeltaIndex(pfi, m => m.Ndcg[i]) == 0);
Assert.True(MinDeltaIndex(pfi, m => m.Ndcg[i]) == 1);
Assert.Equal(0, MaxDeltaIndex(pfi, m => m.Ndcg[i]));
Assert.Equal(1, MinDeltaIndex(pfi, m => m.Ndcg[i]));
}

Done();
Expand All @@ -301,13 +301,13 @@ public void TestPfiRankingOnSparseFeatures()
// For the following metrics higher is better, so minimum delta means more important feature, and vice versa
for (int i = 0; i < pfi[0].Dcg.Length; i++)
{
Assert.True(MaxDeltaIndex(pfi, m => m.Dcg[i]) == 2);
Assert.True(MinDeltaIndex(pfi, m => m.Dcg[i]) == 5);
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.Dcg[i]));
Assert.Equal(5, MinDeltaIndex(pfi, m => m.Dcg[i]));
}
for (int i = 0; i < pfi[0].Ndcg.Length; i++)
{
Assert.True(MaxDeltaIndex(pfi, m => m.Ndcg[i]) == 2);
Assert.True(MinDeltaIndex(pfi, m => m.Ndcg[i]) == 5);
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.Ndcg[i]));
Assert.Equal(5, MinDeltaIndex(pfi, m => m.Ndcg[i]));
}

Done();
Expand All @@ -325,8 +325,8 @@ public void TestPfiClusteringOnDenseFeatures()

var preview = data.Preview();

var model = ML.Clustering.Trainers.KMeans("Features", clustersCount: 5,
advancedSettings: args =>{ args.NormalizeFeatures = NormalizeOption.No;})
var model = ML.Clustering.Trainers.KMeans("Features", clustersCount: 5,
advancedSettings: args =>{ args.NormalizeFeatures = NormalizeOption.No; args.NumThreads = 1; })
Copy link
Contributor Author

@TomFinley TomFinley Dec 19, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is what I am hoping will solve the issue. But, again, it's mostly speculative on my part. I have TBH only seen the spurious failure happen on the build servers for MacOS debug, for which my ability to repro is limited.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am also not sure what is this NormalizeOption.No. I do not think Fit will auto-normalize in any event anyway?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep, I think this argument is only use by the entry point.

In reply to: 242977200 [](ancestors = 242977200)

.Fit(data);
var pfi = ML.Clustering.PermutationFeatureImportance(model, data);

Expand All @@ -336,12 +336,12 @@ public void TestPfiClusteringOnDenseFeatures()
// X3: 2 -- Least important for clustering (smallest range)

// For the following metrics lower is better, so maximum delta means more important feature, and vice versa
Assert.True(MinDeltaIndex(pfi, m => m.AvgMinScore) == 0);
Assert.True(MaxDeltaIndex(pfi, m => m.AvgMinScore) == 2);
Assert.Equal(0, MinDeltaIndex(pfi, m => m.AvgMinScore));
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.AvgMinScore));

// For the following metrics higher is better, so minimum delta means more important feature, and vice versa
Assert.True(MinDeltaIndex(pfi, m => m.Nmi) == 2);
Assert.True(MaxDeltaIndex(pfi, m => m.Nmi) == 0);
Assert.Equal(2, MinDeltaIndex(pfi, m => m.Nmi));
Assert.Equal(0, MaxDeltaIndex(pfi, m => m.Nmi));

Done();
}
Expand Down