Merge performance report, Class Performance Imbalance and Train test …

…Overfit into one check (#364) * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0 * v0
deepchecks · Jan 2, 2022 · a18fc89 · a18fc89
1 parent e537e01
commit a18fc89
Show file tree

Hide file tree

Showing 20 changed files with 3,728 additions and 3,691 deletions.
diff --git a/deepchecks/base/dataset.py b/deepchecks/base/dataset.py
@@ -646,7 +646,7 @@ def classes(self) -> t.List[str]:
             Sorted classes
         """
         if self.label_col is not None:
-            return sorted(self.label_col.unique().tolist())
+            return sorted(self.label_col.dropna().unique().tolist())
         return []
 
     @property

diff --git a/deepchecks/checks/__init__.py b/deepchecks/checks/__init__.py
@@ -25,7 +25,6 @@
 )
 
 from .methodology import (
-    TrainTestDifferenceOverfit,
     BoostingOverfit,
     UnusedFeatures,
     SingleFeatureContribution,
@@ -60,7 +59,6 @@
     SegmentPerformance,
     RegressionSystematicError,
     RegressionErrorDistribution,
-    ClassPerformance,
     MultiModelPerformanceReport,
     ModelErrorAnalysis
 )
@@ -82,7 +80,6 @@
     'LabelAmbiguity',
 
     # methodology checks
-    'TrainTestDifferenceOverfit',
     'BoostingOverfit',
     'UnusedFeatures',
     'SingleFeatureContribution',
@@ -114,7 +111,6 @@
     'SegmentPerformance',
     'RegressionSystematicError',
     'RegressionErrorDistribution',
-    'ClassPerformance',
     'MultiModelPerformanceReport',
     'ModelErrorAnalysis'
 ]
diff --git a/deepchecks/checks/methodology/__init__.py b/deepchecks/checks/methodology/__init__.py
@@ -9,7 +9,6 @@
 # ----------------------------------------------------------------------------
 #
 """Module contains checks for methodological flaws in the model building process."""
-from .performance_overfit import TrainTestDifferenceOverfit
 from .boosting_overfit import BoostingOverfit
 from .unused_features import UnusedFeatures
 from .single_feature_contribution import SingleFeatureContribution
@@ -24,7 +23,6 @@
 
 
 __all__ = [
-    'TrainTestDifferenceOverfit',
     'BoostingOverfit',
     'UnusedFeatures',
     'SingleFeatureContribution',

diff --git a/deepchecks/checks/methodology/performance_overfit.py b/deepchecks/checks/methodology/performance_overfit.py
diff --git a/deepchecks/checks/performance/__init__.py b/deepchecks/checks/performance/__init__.py
@@ -17,7 +17,6 @@
 from .segment_performance import SegmentPerformance
 from .regression_systematic_error import RegressionSystematicError
 from .regression_error_distribution import RegressionErrorDistribution
-from .class_performance import ClassPerformance
 from .model_error_analysis import ModelErrorAnalysis
 
 
@@ -30,7 +29,6 @@
     'SegmentPerformance',
     'RegressionSystematicError',
     'RegressionErrorDistribution',
-    'ClassPerformance',
     'MultiModelPerformanceReport',
     'ModelErrorAnalysis'
 ]
diff --git a/deepchecks/checks/performance/calibration_score.py b/deepchecks/checks/performance/calibration_score.py
@@ -91,7 +91,7 @@ def _calibration_score(self, dataset: Dataset, model):
                     name=f'{class_name} (brier:{briers_scores[class_name]:9.4f})',
                 ))
 
-        fig.update_layout(title_text='Calibration plots  (reliability curve)',
+        fig.update_layout(title_text='Calibration plots (reliability curve)',
                           width=700, height=500)
         fig.update_yaxes(title='Fraction of positives')
         fig.update_xaxes(title='Mean predicted value')