Fixes for running prediction drift & segment performance on various d…

…atasets * Make sure segment performance doesn't try to run on features that are not numeric nor categorical * flatten arrays before series in prediction drift
deepchecks · May 22, 2022 · f374742 · f374742
1 parent ae2c3c4
commit f374742
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 6 deletions.
diff --git a/deepchecks/tabular/checks/model_evaluation/segment_performance.py b/deepchecks/tabular/checks/model_evaluation/segment_performance.py
@@ -100,6 +100,12 @@ def run_logic(self, context: Context, dataset_type: str = 'train') -> CheckResul
             if self.feature_1 not in columns or self.feature_2 not in columns:
                 raise DeepchecksValueError('"feature_1" and "feature_2" must be in dataset columns')
 
+        if self.feature_1 not in (dataset.numerical_features + dataset.cat_features):
+            raise DeepchecksValueError('"feature_1" must be numerical or categorical, but it neither.')
+
+        if self.feature_2 not in (dataset.numerical_features + dataset.cat_features):
+            raise DeepchecksValueError('"feature_2" must be numerical or categorical, but it neither.')
+
         feature_1_filters = partition_column(dataset, self.feature_1, max_segments=self.max_segments)
         feature_2_filters = partition_column(dataset, self.feature_2, max_segments=self.max_segments)
 

diff --git a/deepchecks/tabular/checks/model_evaluation/train_test_prediction_drift.py b/deepchecks/tabular/checks/model_evaluation/train_test_prediction_drift.py
@@ -13,6 +13,7 @@
 import warnings
 from typing import Dict
 
+import numpy as np
 import pandas as pd
 
 from deepchecks import ConditionCategory
@@ -103,12 +104,12 @@ def run_logic(self, context: Context) -> CheckResult:
         test_dataset = context.test
         model = context.model
 
-        train_prediction = model.predict(train_dataset.features_columns)
-        test_prediction = model.predict(test_dataset.features_columns)
+        train_prediction = np.array(model.predict(train_dataset.features_columns))
+        test_prediction = np.array(model.predict(test_dataset.features_columns))
 
         drift_score, method, display = calc_drift_and_plot(
-            train_column=pd.Series(train_prediction),
-            test_column=pd.Series(test_prediction),
+            train_column=pd.Series(train_prediction.flatten()),
+            test_column=pd.Series(test_prediction.flatten()),
             value_name='model predictions',
             column_type='categorical' if train_dataset.label_type == 'classification_label' else 'numerical',
             margin_quantile_filter=self.margin_quantile_filter,

diff --git a/deepchecks/tabular/suites/default_suites.py b/deepchecks/tabular/suites/default_suites.py
@@ -33,6 +33,7 @@
                                        PerformanceReport,
                                        RegressionErrorDistribution,
                                        RegressionSystematicError, RocReport,
+                                       SegmentPerformance,
                                        SimpleModelComparison,
                                        SingleFeatureContribution,
                                        SingleFeatureContributionTrainTest,
@@ -41,6 +42,7 @@
                                        StringMismatchComparison,
                                        TrainTestFeatureDrift,
                                        TrainTestLabelDrift,
+                                       TrainTestPredictionDrift,
                                        TrainTestSamplesMix, UnusedFeatures,
                                        WholeDatasetDrift)
 
@@ -125,8 +127,8 @@ def model_evaluation() -> Suite:
         PerformanceReport().add_condition_train_test_relative_degradation_not_greater_than(),
         RocReport().add_condition_auc_not_less_than(),
         ConfusionMatrixReport(),
-        # SegmentPerformance(),
-        # TrainTestPredictionDrift().add_condition_drift_score_not_greater_than(),
+        SegmentPerformance(),
+        TrainTestPredictionDrift().add_condition_drift_score_not_greater_than(),
         SimpleModelComparison().add_condition_gain_not_less_than(),
         ModelErrorAnalysis().add_condition_segments_performance_relative_difference_not_greater_than(),
         CalibrationScore(),

diff --git a/tests/tabular/checks/model_evaluation/segment_performance_test.py b/tests/tabular/checks/model_evaluation/segment_performance_test.py
@@ -67,6 +67,17 @@ def test_segment_performance_illegal_features(diabetes_split_dataset_and_model):
     )
 
 
+def test_segment_performance_non_cat_or_num(city_arrogance_split_dataset_and_model):
+    # Arrange
+    _, val, model = city_arrogance_split_dataset_and_model
+
+    # Act & Assert
+    assert_that(
+        calling(SegmentPerformance(feature_1='city', feature_2='sex').run).with_args(val, model),
+        raises(DeepchecksValueError, r'\"feature_1\" must be numerical or categorical, but it neither.')
+    )
+
+
 def test_segment_top_features(diabetes_split_dataset_and_model):
     # Arrange
     _, val, model = diabetes_split_dataset_and_model