fix errors related to string labels (#501)

* v0

* linting

* better_transform

* linting

* added_tests

* added_tests

* renames

* renames

deepchecks/checks/performance/calibration_score.py

@@ -52,7 +52,7 @@ def _calibration_score(self, dataset: Dataset, model):
         briers_scores = {}
 
         if len(dataset.classes) == 2:
-            briers_scores[0] = brier_score_loss(ds_y, y_pred[:, 1])
+            briers_scores[0] = brier_score_loss(ds_y, y_pred[:, 1], pos_label=dataset.classes[1])
         else:
             for class_index, class_name in enumerate(dataset.classes):
                 prob_pos = y_pred[:, class_index]

deepchecks/checks/performance/model_error_analysis.py

@@ -21,6 +21,7 @@
 from sklearn.metrics import r2_score
 from sklearn.pipeline import Pipeline
 from sklearn.tree import DecisionTreeRegressor
+from sklearn import preprocessing
 
 from deepchecks import Dataset, CheckResult, TrainTestBaseCheck, ConditionResult, ConditionCategory
 from deepchecks.errors import DeepchecksProcessError
@@ -147,11 +148,15 @@ def run(self, train_dataset: Dataset, test_dataset: Dataset, model=None) -> Chec
 
         # Create scoring function, used to calculate the per sample model error
         if task_type == ModelType.REGRESSION:
-            def scoring_func(dataset):
+            def scoring_func(dataset: Dataset):
                 return per_sample_mse(dataset.label_col, model.predict(dataset.features_columns))
         else:
-            def scoring_func(dataset):
-                return per_sample_binary_cross_entropy(dataset.label_col,
+            le = preprocessing.LabelEncoder()
+            le.fit(train_dataset.classes)
+
+            def scoring_func(dataset: Dataset):
+                encoded_label = le.transform(dataset.label_col)
+                return per_sample_binary_cross_entropy(encoded_label,
                                                        model.predict_proba(dataset.features_columns))
 
         train_scores = scoring_func(train_dataset)

tests/checks/performance/calibration_score_test.py

@@ -65,15 +65,27 @@ def test_model_info_object(iris_labeled_dataset, iris_adaboost):
         2: close_to(0.28, 0.05)
     }))
 
-
 def test_binary_model_info_object(iris_dataset_single_class_labeled, iris_random_forest_single_class):
     # Arrange
     check = CalibrationScore()
     # Act X
     result = check.run(iris_dataset_single_class_labeled, iris_random_forest_single_class).value
     # Assert
-    assert len(result) == 1  # iris has 3 targets
+    assert len(result) == 1
 
     assert_that(result, has_entries({
         0: close_to(0.0002, 0.0005)
     }))
+
+def test_binary_string_model_info_object(iris_binary_string_split_dataset_and_model):
+    # Arrange
+    _, test_ds, clf = iris_binary_string_split_dataset_and_model
+    check = CalibrationScore()
+    # Act X
+    result = check.run(test_ds, clf).value
+    # Assert
+    assert len(result) == 1
+
+    assert_that(result, has_entries({
+        0: close_to(0.04, 0.001)
+    }))

tests/checks/performance/model_error_analysis_test.py

@@ -50,6 +50,15 @@ def test_model_error_analysis_classification(iris_labeled_dataset, iris_adaboost
     assert_that(result_value['feature_segments']['petal length (cm)'], has_length(2))
 
 
+def test_binary_string_model_info_object(iris_binary_string_split_dataset_and_model):
+    # Arrange
+    train_ds, test_ds, clf = iris_binary_string_split_dataset_and_model
+    check = ModelErrorAnalysis()
+    # Act X
+    result_value = check.run(train_ds, test_ds, clf).value
+    # Assert
+    assert_that(result_value['feature_segments']['petal length (cm)'], has_length(2))
+
 def test_condition_fail(iris_labeled_dataset, iris_adaboost):
     # Act
     check_result = ModelErrorAnalysis().add_condition_segments_performance_relative_difference_not_greater_than(

tests/conftest.py

@@ -173,6 +173,20 @@ def iris_split_dataset_and_model_rf(iris) -> Tuple[Dataset, Dataset, RandomFores
     return train_ds, val_ds, clf
 
 
+@pytest.fixture
+def iris_binary_string_split_dataset_and_model(iris) -> Tuple[Dataset, Dataset, DecisionTreeClassifier]:
+    """Return Iris train and test datasets and trained DecisionTreeClassifier model."""
+    iris = iris.copy()
+    iris.loc[iris['target'] != 2, 'target'] = 'a'
+    iris.loc[iris['target'] == 2, 'target'] = 'b'
+    train, test = train_test_split(iris, test_size=0.33, random_state=42)
+    train_ds = Dataset(train, label='target')
+    test_ds = Dataset(test, label='target')
+    clf = DecisionTreeClassifier(random_state=0)
+    clf.fit(train_ds.features_columns, train_ds.label_col)
+    return train_ds, test_ds, clf
+
+
 # NaN dataframes:
 @pytest.fixture(scope='session')
 def df_with_nan_row():