diff --git a/README.md b/README.md index 9d7595cc5..7560cc4e2 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ install.packages("DALEX") The **Python** version of dalex is available on [pip](https://pypi.org/project/dalex/) ```console -pip install dalex +pip install dalex -U ``` ## Learn more diff --git a/python/dalex/NEWS.md b/python/dalex/NEWS.md index 11dd953a6..20457697e 100644 --- a/python/dalex/NEWS.md +++ b/python/dalex/NEWS.md @@ -1,6 +1,17 @@ dalex (development) ---------------------------------------------------------------- -* ... + +### bug fixes + +* `ModelPerformance.plot` now uses a drwhy color palette + +### features + +* added the `ResidualDiagnostics` object with a `plot` method +* added `model_diagnostics` method to the `Explainer`, which performs residual diagnostics +* added `predict_surrogate` method to the `Explainer`, which is a wrapper for the `lime` + tabular explanation from the [lime](https://github.com/marcotcr/lime) package +* added a `__str__` method to all of the explanation objects (it prints the `result` attribute) dalex 0.2.0 ---------------------------------------------------------------- diff --git a/python/dalex/README.md b/python/dalex/README.md index ee9fd90f4..32a7b80a5 100644 --- a/python/dalex/README.md +++ b/python/dalex/README.md @@ -27,7 +27,7 @@ The `dalex` package is a part of [DrWhy.AI](http://DrWhy.AI) universe. ## Installation ```console -pip install dalex==0.1.9 +pip install dalex -U ``` ## Resources diff --git a/python/dalex/dalex/_explainer/checks.py b/python/dalex/dalex/_explainer/checks.py index c82285142..947fa9741 100644 --- a/python/dalex/dalex/_explainer/checks.py +++ b/python/dalex/dalex/_explainer/checks.py @@ -1,5 +1,6 @@ import numpy as np import pandas as pd +from copy import deepcopy from .helper import verbose_cat, is_y_in_data from .yhat import * @@ -290,3 +291,30 @@ def check_loss_function(explainer, loss_function): def check_model_type(model_type, model_type_): return model_type_ if model_type is None else model_type + + +def check_new_observation_lime(new_observation): + # lime accepts only np.array as data_row + + new_observation_ = deepcopy(new_observation) + if isinstance(new_observation_, pd.Series): + new_observation_ = new_observation_.to_numpy() + elif isinstance(new_observation_, np.ndarray): + if new_observation_.ndim == 2: + if new_observation.shape[0] != 1: + raise ValueError("Wrong new_observation dimension") + # make 2D array 1D + new_observation_ = new_observation_.flatten() + elif new_observation_.ndim > 2: + raise ValueError("Wrong new_observation dimension") + elif isinstance(new_observation_, list): + new_observation_ = np.array(new_observation_) + elif isinstance(new_observation_, pd.DataFrame): + if new_observation.shape[0] != 1: + raise ValueError("Wrong new_observation dimension") + else: + new_observation_ = new_observation.to_numpy().flatten() + else: + raise TypeError("new_observation must be a list or numpy.ndarray or pandas.Series or pandas.DataFrame") + + return new_observation_ diff --git a/python/dalex/dalex/_explainer/helper.py b/python/dalex/dalex/_explainer/helper.py index 7c79c3e8a..05ce53d88 100644 --- a/python/dalex/dalex/_explainer/helper.py +++ b/python/dalex/dalex/_explainer/helper.py @@ -15,3 +15,33 @@ def is_y_in_data(data, y): def get_model_info(model): model_package = re.search("(?<==4.9.0', 'tqdm>=4.42.1' ], + test_requirements=[ + 'lime>=0.2.0.1', # Explainer.predict_surrogate + 'statsmodels>=0.11.1' # LOWESS trendlines in ResidualDiagnostics.plot + ], packages=setuptools.find_packages(include=["dalex", "dalex.*"]), python_requires='>=3.6', include_package_data=True diff --git a/python/dalex/test/test_aggregated_profiles.py b/python/dalex/test/test_aggregated_profiles.py index b4d94c9ff..8872000cf 100644 --- a/python/dalex/test/test_aggregated_profiles.py +++ b/python/dalex/test/test_aggregated_profiles.py @@ -14,7 +14,7 @@ import dalex as dx -class APTestTitanic(unittest.TestCase): +class AggregatedProfilesTestTitanic(unittest.TestCase): def setUp(self): data = dx.datasets.load_titanic() data.loc[:, 'survived'] = LabelEncoder().fit_transform(data.survived) diff --git a/python/dalex/test/test_ceteris_paribus.py b/python/dalex/test/test_ceteris_paribus.py index 3830df8e2..83235e797 100644 --- a/python/dalex/test/test_ceteris_paribus.py +++ b/python/dalex/test/test_ceteris_paribus.py @@ -14,7 +14,6 @@ class CeterisParibusTestTitanic(unittest.TestCase): - def setUp(self): data = dx.datasets.load_titanic() data.loc[:, 'survived'] = LabelEncoder().fit_transform(data.survived) diff --git a/python/dalex/test/test_model_diagnostics.py b/python/dalex/test/test_model_diagnostics.py new file mode 100644 index 000000000..b1198085b --- /dev/null +++ b/python/dalex/test/test_model_diagnostics.py @@ -0,0 +1,83 @@ +import unittest + +import numpy as np +import pandas as pd +from sklearn.compose import ColumnTransformer +from sklearn.impute import SimpleImputer +from sklearn.neural_network import MLPClassifier +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import StandardScaler, LabelEncoder, OneHotEncoder + +import dalex as dx +from plotly.graph_objs import Figure + + +class ModelDiagnosticsTestTitanic(unittest.TestCase): + def setUp(self): + data = dx.datasets.load_titanic() + data.loc[:, 'survived'] = LabelEncoder().fit_transform(data.survived) + + self.X = data.drop(columns='survived') + self.y = data.survived + + numeric_features = ['age', 'fare', 'sibsp', 'parch'] + numeric_transformer = Pipeline(steps=[ + ('imputer', SimpleImputer(strategy='median')), + ('scaler', StandardScaler())]) + + categorical_features = ['gender', 'class', 'embarked'] + categorical_transformer = Pipeline(steps=[ + ('imputer', SimpleImputer(strategy='constant', fill_value='missing')), + ('onehot', OneHotEncoder(handle_unknown='ignore'))]) + + preprocessor = ColumnTransformer( + transformers=[ + ('num', numeric_transformer, numeric_features), + ('cat', categorical_transformer, categorical_features)]) + + clf = Pipeline(steps=[('preprocessor', preprocessor), + ('classifier', MLPClassifier(hidden_layer_sizes=(50, 100, 50), + max_iter=400, random_state=0))]) + + clf.fit(self.X, self.y) + + self.exp = dx.Explainer(clf, self.X, self.y, verbose=False) + self.exp2 = dx.Explainer(clf, self.X, self.y, label="model2", verbose=False) + + def test_constructor(self): + case1 = self.exp.model_diagnostics() + self.assertIsInstance(case1, (dx.dataset_level.ResidualDiagnostics,)) + self.assertIsInstance(case1.result, (pd.DataFrame,)) + self.assertEqual(case1.result.shape[0], self.exp.data.shape[0]) + self.assertTrue(np.isin(['y', 'y_hat', 'residuals', 'abs_residuals', 'label', 'ids'], + case1.result.columns).all()) + + case2 = self.exp.model_diagnostics(variables=['age', 'class']) + self.assertIsInstance(case2, (dx.dataset_level.ResidualDiagnostics,)) + self.assertIsInstance(case2.result, (pd.DataFrame,)) + self.assertEqual(case2.result.shape[0], self.exp.data.shape[0]) + self.assertTrue(np.isin(['y', 'y_hat', 'residuals', 'abs_residuals', 'label', 'ids', 'age', 'class'], + case2.result.columns).all()) + self.assertFalse(np.isin(['fare', 'sibsp', 'gender', 'embarked'], case2.result.columns).any()) + + def test_plot(self): + + case1 = self.exp.model_diagnostics(variables=['fare', 'embarked']) + case2 = self.exp.model_diagnostics() + case3 = self.exp2.model_diagnostics() + + self.assertIsInstance(case1, dx.dataset_level.ResidualDiagnostics) + self.assertIsInstance(case2, dx.dataset_level.ResidualDiagnostics) + self.assertIsInstance(case3, dx.dataset_level.ResidualDiagnostics) + + fig1 = case1.plot(title="test1", variable="fare", show=False) + fig2 = case2.plot(case3, variable="sibsp", yvariable="abs_residuals", show=False) + fig3 = case2.plot(smooth=False, line_width=6, marker_size=1, variable="age", show=False) + + self.assertIsInstance(fig1, Figure) + self.assertIsInstance(fig2, Figure) + self.assertIsInstance(fig3, Figure) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/dalex/test/test_model_performance.py b/python/dalex/test/test_model_performance.py index 8c902316a..8dd180f7d 100644 --- a/python/dalex/test/test_model_performance.py +++ b/python/dalex/test/test_model_performance.py @@ -45,17 +45,17 @@ def setUp(self): self.exp2 = dx.Explainer(clf, self.X, self.y, label="model2", verbose=False) def test_constructor(self): - self.assertIsInstance(self.exp.model_performance('classification'), (dx.dataset_level.ModelPerformance,)) - self.assertIsInstance(self.exp.model_performance('classification').result, (pd.DataFrame,)) - self.assertEqual(self.exp.model_performance('classification').result.shape[0], 1) - self.assertTrue(np.isin(['recall', 'precision', 'f1', 'accuracy', 'auc'], - self.exp.model_performance('classification').result.columns).all()) - - self.assertIsInstance(self.exp.model_performance('regression'), (dx.dataset_level.ModelPerformance,)) - self.assertIsInstance(self.exp.model_performance('regression').result, (pd.DataFrame,)) - self.assertEqual(self.exp.model_performance('regression').result.shape[0], 1) - self.assertTrue(np.isin(['mse', 'rmse', 'r2', 'mae', 'mad'], - self.exp.model_performance('regression').result.columns).all()) + case1 = self.exp.model_performance('classification') + self.assertIsInstance(case1, (dx.dataset_level.ModelPerformance,)) + self.assertIsInstance(case1.result, (pd.DataFrame,)) + self.assertEqual(case1.result.shape[0], 1) + self.assertTrue(np.isin(['recall', 'precision', 'f1', 'accuracy', 'auc'], case1.result.columns).all()) + + case2 = self.exp.model_performance('regression') + self.assertIsInstance(case2, (dx.dataset_level.ModelPerformance,)) + self.assertIsInstance(case2.result, (pd.DataFrame,)) + self.assertEqual(case2.result.shape[0], 1) + self.assertTrue(np.isin(['mse', 'rmse', 'r2', 'mae', 'mad'], case2.result.columns).all()) def test_plot(self): diff --git a/python/dalex/test/test_predict_surrogate.py b/python/dalex/test/test_predict_surrogate.py new file mode 100644 index 000000000..4c13a4e50 --- /dev/null +++ b/python/dalex/test/test_predict_surrogate.py @@ -0,0 +1,63 @@ +import unittest + +from sklearn.neural_network import MLPClassifier +from sklearn.ensemble import RandomForestRegressor +from sklearn.preprocessing import LabelEncoder + +import dalex as dx +import lime + +class PredictSurrogateTestTitanic(unittest.TestCase): + def setUp(self): + data = dx.datasets.load_titanic() + self.X = data.drop(columns=['survived', 'class', 'embarked']) + self.y = data.survived + self.X.gender = LabelEncoder().fit_transform(self.X.gender) + + model = MLPClassifier(hidden_layer_sizes=(50, 50), max_iter=400, random_state=0) + model.fit(self.X, self.y) + self.exp = dx.Explainer(model, self.X, self.y, verbose=False) + + data2 = dx.datasets.load_fifa() + self.X2 = data2.drop(["nationality", "overall", "potential", + "value_eur", "wage_eur"], axis=1).iloc[0:2000, 0:10] + self.y2 = data2['value_eur'].iloc[0:2000] + + model2 = RandomForestRegressor(random_state=0) + model2.fit(self.X2, self.y2) + self.exp2 = dx.Explainer(model2, self.X2, self.y2, verbose=False) + + def test(self): + case1 = self.exp.predict_surrogate(new_observation=self.X.iloc[1, :], + feature_names=self.X.columns) + case2 = self.exp.predict_surrogate(new_observation=self.X.iloc[1:2, :], + mode='classification', + feature_names=self.X.columns, + discretize_continuous=True, + num_features=4) + case3 = self.exp.predict_surrogate(new_observation=self.X.iloc[1:2, :].to_numpy(), + feature_names=self.X.columns, + kernel_width=2, + num_samples=50) + case4 = self.exp2.predict_surrogate(new_observation=self.X2.iloc[1, :], + feature_names=self.X2.columns) + case5 = self.exp2.predict_surrogate(new_observation=self.X2.iloc[1:2, :], + mode='regression', + feature_names=self.X2.columns, + discretize_continuous=True, + num_features=4) + case6 = self.exp2.predict_surrogate(new_observation=self.X2.iloc[1:2, :].to_numpy(), + feature_names=self.X2.columns, + kernel_width=2, + num_samples=50) + + self.assertIsInstance(case1, lime.explanation.Explanation) + self.assertIsInstance(case2, lime.explanation.Explanation) + self.assertIsInstance(case3, lime.explanation.Explanation) + self.assertIsInstance(case4, lime.explanation.Explanation) + self.assertIsInstance(case5, lime.explanation.Explanation) + self.assertIsInstance(case6, lime.explanation.Explanation) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/dalex/test/test_variable_importance.py b/python/dalex/test/test_variable_importance.py index c97141a36..0c478a6b2 100644 --- a/python/dalex/test/test_variable_importance.py +++ b/python/dalex/test/test_variable_importance.py @@ -128,14 +128,14 @@ def test_calculate_variable_importance(self): vi[0].columns).all()) def test_constructor(self): - self.assertIsInstance(self.exp.model_parts(), (dx.dataset_level.VariableImportance,)) - self.assertIsInstance(self.exp.model_parts().result, (pd.DataFrame,)) - self.assertEqual(list(self.exp.model_parts().result.columns), - ['variable', 'dropout_loss', 'label']) - - vi = self.exp.model_parts(keep_raw_permutations=True) - self.assertTrue(hasattr(vi, 'permutation')) - self.assertIsInstance(vi.permutation, pd.DataFrame) + case1 = self.exp.model_parts() + self.assertIsInstance(case1, (dx.dataset_level.VariableImportance,)) + self.assertIsInstance(case1.result, (pd.DataFrame,)) + self.assertEqual(list(case1.result.columns), ['variable', 'dropout_loss', 'label']) + + case2 = self.exp.model_parts(keep_raw_permutations=True) + self.assertTrue(hasattr(case2, 'permutation')) + self.assertIsInstance(case2.permutation, pd.DataFrame) def test_variables_and_variable_groups(self): diff --git a/tox.ini b/tox.ini index ac05912be..3d02bc9ba 100644 --- a/tox.ini +++ b/tox.ini @@ -23,3 +23,5 @@ commands = discover deps = discover scikit-learn + lime + statsmodels