static_model_suites_fix (#1558)

* static_model_suites_fix

* linting

* linting

* 36_fix

deepchecks/tabular/checks/model_evaluation/unused_features.py

@@ -22,9 +22,9 @@
 from sklearn.preprocessing import OrdinalEncoder, RobustScaler
 
 from deepchecks.core import CheckResult, ConditionCategory, ConditionResult
+from deepchecks.core.errors import DeepchecksValueError
 from deepchecks.tabular import Context, Dataset, TrainTestCheck
 from deepchecks.utils.function import run_available_kwargs
-from deepchecks.utils.typing import BasicModel
 
 __all__ = ['UnusedFeatures']
 
@@ -72,44 +72,6 @@ def __init__(
         self.n_top_unused_to_show = n_top_unused_to_show
         self.random_state = random_state
 
-    def run(self,
-            train_dataset: Dataset,
-            test_dataset: Dataset,
-            model: BasicModel = None,
-            feature_importance_force_permutation: bool = False,
-            feature_importance_timeout: int = None) -> CheckResult:
-        """Run the check.
-
-        Parameters
-        ----------
-        train_dataset : Dataset
-            dataset representing data an estimator was fitted on
-        test_dataset : Dataset
-            dataset representing data an estimator predicts on
-        model : BasicModel
-            A scikit-learn-compatible fitted estimator instance
-        feature_importance_force_permutation : bool , default: False
-            force calculation of permutation features importance
-        feature_importance_timeout : int , default: None
-            timeout in second for the permutation features importance calculation
-
-        Returns
-        -------
-        CheckResult
-            value is a dataframe with metrics as indexes, and scores per training and test in the columns.
-            display data is a bar graph of the metrics for training and test data.
-
-        Raises
-        ------
-        DeepchecksValueError
-            If neither train dataset nor test dataset exist, or either of the dataset objects are
-            not a Dataset instance with a label.
-        """
-        c = Context(train_dataset, test_dataset, model,
-                    feature_importance_force_permutation=feature_importance_force_permutation,
-                    feature_importance_timeout=feature_importance_timeout)
-        return self.run_logic(c)
-
     def run_logic(self, context: Context) -> CheckResult:
         """Run check."""
         if context.have_test():
@@ -119,6 +81,8 @@ def run_logic(self, context: Context) -> CheckResult:
         _ = context.model  # validate model
 
         feature_importance = context.features_importance
+        if feature_importance is None:
+            raise DeepchecksValueError('Feature Importance is not available.')
         dataset.assert_features()
 
         # Calculate normalized variance per feature based on PCA decomposition

deepchecks/tabular/context.py

@@ -127,6 +127,9 @@ def _predict_proba(self, data: pd.DataFrame):
         self._validate_data(data)
         return self.probas.loc[data.index].to_numpy()
 
+    def fit(self, *args, **kwargs):
+        """Just for python 3.6 (sklearn validates fit method)."""
+
 
 class Context:
     """Contains all the data + properties the user has passed to a check/suite, and validates it seamlessly.

deepchecks/tabular/suite.py

@@ -12,6 +12,7 @@
 # pylint: disable=broad-except
 from typing import Callable, Mapping, Optional, Tuple, Union
 
+import numpy as np
 import pandas as pd
 
 from deepchecks.core.check_result import CheckFailure
@@ -43,7 +44,11 @@ def run(
             feature_importance_force_permutation: bool = False,
             feature_importance_timeout: int = None,
             scorers: Mapping[str, Union[str, Callable]] = None,
-            scorers_per_class: Mapping[str, Union[str, Callable]] = None
+            scorers_per_class: Mapping[str, Union[str, Callable]] = None,
+            y_pred_train: np.ndarray = None,
+            y_pred_test: np.ndarray = None,
+            y_proba_train: np.ndarray = None,
+            y_proba_test: np.ndarray = None,
     ) -> SuiteResult:
         """Run all checks.
 
@@ -68,6 +73,14 @@ def run(
             See <a href=
             "https://scikit-learn.org/stable/modules/model_evaluation.html#from-binary-to-multiclass-and-multilabel">
             scikit-learn docs</a>
+        y_pred_train: np.ndarray , default: None
+            Array of the model prediction over the train dataset.
+        y_pred_test: np.ndarray , default: None
+            Array of the model prediction over the test dataset.
+        y_proba_train: np.ndarray , default: None
+            Array of the model prediction probabilities over the train dataset.
+        y_proba_test: np.ndarray , default: None
+            Array of the model prediction probabilities over the test dataset.
         Returns
         -------
         SuiteResult
@@ -78,7 +91,9 @@ def run(
                           feature_importance_force_permutation=feature_importance_force_permutation,
                           feature_importance_timeout=feature_importance_timeout,
                           scorers=scorers,
-                          scorers_per_class=scorers_per_class)
+                          scorers_per_class=scorers_per_class,
+                          y_pred_train=y_pred_train, y_pred_test=y_pred_test,
+                          y_proba_train=y_proba_train, y_proba_test=y_proba_test)
 
         progress_bar = create_progress_bar(
             iterable=list(self.checks.values()),

deepchecks/vision/suite.py

@@ -11,7 +11,7 @@
 """Module for base vision abstractions."""
 # pylint: disable=broad-except,not-callable
 from collections import OrderedDict
-from typing import Dict, Mapping, Optional, Tuple, Union
+from typing import Dict, List, Mapping, Optional, Tuple, Union
 
 import torch
 from ignite.metrics import Metric
@@ -48,6 +48,8 @@ def run(
             device: Union[str, torch.device, None] = 'cpu',
             random_state: int = 42,
             n_samples: Optional[int] = 10_000,
+            train_predictions: Union[List[torch.Tensor], torch.Tensor] = None,
+            test_predictions: Union[List[torch.Tensor], torch.Tensor] = None,
     ) -> SuiteResult:
         """Run all checks.
 
@@ -96,7 +98,9 @@ def run(
                     scorers_per_class=scorers_per_class,
                     device=device,
                     random_state=random_state,
-                    n_samples=n_samples
+                    n_samples=n_samples,
+                    train_predictions=train_predictions,
+                    test_predictions=test_predictions,
                 )
 
             # Initialize here all the checks that are not single dataset,

tests/tabular/test_dummy_model.py

@@ -21,11 +21,15 @@
 from deepchecks.tabular.checks.model_evaluation.regression_error_distribution import RegressionErrorDistribution
 from deepchecks.tabular.checks.model_evaluation.roc_report import RocReport
 from deepchecks.tabular.checks.model_evaluation.simple_model_comparison import SimpleModelComparison
+from deepchecks.tabular.checks.model_evaluation.unused_features import UnusedFeatures
 from deepchecks.tabular.context import Context
 from deepchecks.tabular.dataset import Dataset
 from tests.base.utils import equal_condition_result
 from tests.tabular.checks.model_evaluation.simple_model_comparison_test import assert_regression
 
+from deepchecks.tabular.suites.default_suites import full_suite
+from tests.conftest import get_expected_results_length, validate_suite_result
+
 
 def _dummify_model(train, test, model):
     y_pred_train = y_pred_test = y_proba_train = y_proba_test = None
@@ -178,3 +182,16 @@ def test_bad_pred_proba(iris_labeled_dataset, iris_adaboost):
             ValidationError,
             r'Prediction propabilities array didn\'t match predictions result')
     )
+
+
+def test_suite(diabetes_split_dataset_and_model):
+    train, test, clf = diabetes_split_dataset_and_model
+    y_pred_train, y_pred_test, y_proba_train, y_proba_test = _dummify_model(train, test, clf)
+
+    args = dict(train_dataset=train, test_dataset=test,
+                y_pred_train=y_pred_train, y_pred_test=y_pred_test,
+                y_proba_train=y_proba_train, y_proba_test=y_proba_test)
+    suite = full_suite()
+    result = suite.run(**args)
+    length = get_expected_results_length(suite, args)
+    validate_suite_result(result, length)

tests/vision/base/test_static_predictions.py

@@ -16,6 +16,7 @@
 from hamcrest import (assert_that, calling, close_to, equal_to, has_entries, has_items, has_length, has_properties,
                       has_property, instance_of, is_, raises)
 
+from deepchecks.vision.suites.default_suites import full_suite
 from deepchecks.core.check_result import CheckResult
 from deepchecks.vision.base_checks import SingleDatasetCheck
 from deepchecks.vision.batch_wrapper import Batch
@@ -27,6 +28,8 @@
 from deepchecks.vision.vision_data import VisionData
 from tests.base.utils import equal_condition_result
 
+from tests.conftest import get_expected_results_length, validate_suite_result
+
 
 class _StaticPred(SingleDatasetCheck):
     def initialize_run(self, context: Context, dataset_kind):
@@ -168,3 +171,17 @@ def test_train_test_prediction_with_drift_object_detection_change_max_cat(coco_t
         )
         }
     ))
+
+
+def test_suite(coco_train_visiondata, coco_test_visiondata,
+               mock_trained_yolov5_object_detection, device):
+    train_preds, test_preds = _create_static_predictions(coco_train_visiondata,
+                                                         coco_test_visiondata,
+                                                         mock_trained_yolov5_object_detection)
+
+    args = dict(train_dataset=coco_train_visiondata, test_dataset=coco_test_visiondata,
+                train_predictions=train_preds, test_predictions=test_preds)
+    suite = full_suite()
+    result = suite.run(**args)
+    length = get_expected_results_length(suite, args)
+    validate_suite_result(result, length)