Skip to content

Commit

Permalink
added scores to fairness metrics calculations (#65)
Browse files Browse the repository at this point in the history
* fixed fairness metrics calculations based on scores (Theil/Entropy)
  • Loading branch information
jakubczakon committed Oct 22, 2019
1 parent 84a4309 commit a339a96
Show file tree
Hide file tree
Showing 4 changed files with 24 additions and 20 deletions.
2 changes: 1 addition & 1 deletion docs/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
# The short X.Y version
version = '0.13'
# The full version, including alpha/beta/rc tags
release = '0.13.8'
release = '0.13.9'

# -- General configuration ---------------------------------------------------

Expand Down
6 changes: 4 additions & 2 deletions docs/examples/log_fairness_metrics.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@
"clf.fit(X_train, y_train)\n",
"\n",
"y_test_pred = clf.predict_proba(X_test)\n",
"test['recid_prediction'] = (y_test_pred[:,1] >0.5).astype(int)\n",
"test['recid_prediction_score'] = y_test_pred[:,1]\n",
"test['recid_prediction_class'] = (test['recid_prediction_score'] >0.5).astype(int)\n",
"\n",
"roc_auc = roc_auc_score(y_test, y_test_pred[:,1])"
]
Expand Down Expand Up @@ -102,7 +103,8 @@
"\n",
"with neptune.create_experiment()\n",
" neptune.log_metric('roc_auc',roc_auc)\n",
" log_fairness_classification_metrics(test['two_year_recid'], test['recid_prediction'], test[['race']],\n",
" log_fairness_classification_metrics(test['two_year_recid'], test['recid_prediction_class'], \n",
" test['recid_prediction_score'], test[['race']],\n",
" favorable_label=0, unfavorable_label=1,\n",
" privileged_groups={'race':[3]}, unprivileged_groups={'race':[1,2,4,5,6]})"
]
Expand Down
34 changes: 18 additions & 16 deletions neptunecontrib/monitoring/fairness.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
from neptunecontrib.monitoring.utils import send_figure


def log_fairness_classification_metrics(y_true, y_pred_class, sensitive_attributes,
def log_fairness_classification_metrics(y_true, y_pred_class, y_pred_score, sensitive_attributes,
favorable_label, unfavorable_label,
privileged_groups, unprivileged_groups,
experiment=None, prefix=''):
Expand All @@ -47,6 +47,7 @@ def log_fairness_classification_metrics(y_true, y_pred_class, sensitive_attribut
Args:
y_true (array-like, shape (n_samples)): Ground truth (correct) target values.
y_pred_class (array-like, shape (n_samples)): Class predictions with values 0 or 1.
y_pred_score (array-like, shape (n_samples)): Class predictions with values from 0 to 1. Default None.
sensitive_attributes (pandas.DataFrame, shape (n_samples, k)): datafame containing only sensitive columns.
favorable_label (str or int): label that is favorable, brings positive value to a person being classified.
unfavorable_label (str or int): label that is unfavorable, brings positive value to a person being classified.
Expand All @@ -66,7 +67,7 @@ def log_fairness_classification_metrics(y_true, y_pred_class, sensitive_attribut
neptune.init()
with neptune.create_experiment():
log_fairness_classification_metrics(y_test, y_test_pred_class, test[['race']],
log_fairness_classification_metrics(y_true, y_pred_class, y_pred_score, test[['race']],
favorable_label='granted_parole',
unfavorable_label='not_granted_parole',
privileged_groups={'race':['Caucasian']},
Expand All @@ -84,14 +85,8 @@ def log_fairness_classification_metrics(y_true, y_pred_class, sensitive_attribut

privileged_info = _fmt_priveleged_info(privileged_groups, unprivileged_groups)

data = pd.DataFrame()
data['ground_truth'] = y_true.values
data['prediction'] = y_pred_class.values
for col in sensitive_attributes.columns:
data[col] = sensitive_attributes[col].values

ground_truth_test = _make_dataset(data, 'ground_truth', **bias_info, **privileged_info)
prediction_test = _make_dataset(data, 'prediction', **bias_info, **privileged_info)
ground_truth_test = _make_dataset(sensitive_attributes, y_true, **bias_info, **privileged_info)
prediction_test = _make_dataset(sensitive_attributes, y_pred_class, y_pred_score, **bias_info, **privileged_info)

clf_metric = ClassificationMetric(ground_truth_test, prediction_test, **privileged_info)

Expand All @@ -113,13 +108,20 @@ def log_fairness_classification_metrics(y_true, y_pred_class, sensitive_attribut
plt.close()


def _make_dataset(data, outcome, protected_columns,
privileged_groups, unprivileged_groups,
favorable_label, unfavorable_label):
df = data.copy()
df['outcome'] = data[outcome].values
def _make_dataset(features, labels, scores=None, protected_columns=None,
privileged_groups=None, unprivileged_groups=None,
favorable_label=None, unfavorable_label=None):
df = features.copy()
df['outcome'] = labels

if scores is not None:
scores_names = 'scores'
df[scores_names] = scores
else:
scores_names = []

dataset = BinaryLabelDataset(df=df, label_names=['outcome'], protected_attribute_names=protected_columns,
dataset = BinaryLabelDataset(df=df, label_names=['outcome'], scores_names=scores_names,
protected_attribute_names=protected_columns,
favorable_label=favorable_label, unfavorable_label=unfavorable_label,
unprivileged_protected_attributes=unprivileged_groups)
return dataset
Expand Down
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ def main():

setup(
name='neptune-contrib',
version='0.13.8',
version='0.13.9',
description='Neptune.ml contributions library',
author='neptune.ml',
support='contact@neptune.ml',
Expand Down

0 comments on commit a339a96

Please sign in to comment.